Files

639 lines
17 KiB
Python

import asyncio
import hashlib
import io
import json
import os
import random
import sys
import tempfile
import time
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
import aio_pika
from minio import Minio
from PIL import Image
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
from app.models.attachment import Attachment
from app.models.user import User
from app.models.chat import Chat, ChatMember
from app.config import (
DATABASE_URL,
S3_ENDPOINT,
S3_ACCESS_KEY_ID,
S3_SECRET_ACCESS_KEY,
S3_BUCKET,
)
RABBITMQ_URL = os.getenv(
"RABBITMQ_URL",
"amqp://guest:guest@rabbitmq:5672/",
)
QUEUE_NAME = "image_tasks"
MAX_ATTEMPTS = max(
1, int(os.getenv("WORKER_MAX_ATTEMPTS", "5"))
)
CONCURRENCY = max(
1, int(os.getenv("WORKER_CONCURRENCY", "1"))
)
PREFETCH = max(
1, int(os.getenv("WORKER_PREFETCH", "1"))
)
MAX_IMAGE_BYTES = 10 * 1024 * 1024
PROCESS_TIMEOUT_SECONDS = 30
LEASE_SECONDS = 300
endpoint = S3_ENDPOINT.replace(
"http://", ""
).replace(
"https://", ""
)
minio_client = Minio(
endpoint,
access_key=S3_ACCESS_KEY_ID,
secret_key=S3_SECRET_ACCESS_KEY,
secure=False,
)
engine = create_async_engine(DATABASE_URL)
async_session_maker = async_sessionmaker(
engine,
expire_on_commit=False,
)
class RetryableProcessingError(Exception):
def __init__(self, error_code: str):
self.error_code = error_code
super().__init__(error_code)
class PermanentProcessingError(Exception):
def __init__(self, error_code: str):
self.error_code = error_code
super().__init__(error_code)
class LeaseBusyError(Exception):
pass
class LeaseLostError(Exception):
pass
def get_rss_mb():
try:
with open("/proc/self/status") as file:
for line in file:
if line.startswith("VmRSS:"):
kb = int(line.split()[1])
return round(kb / 1024, 2)
except (OSError, ValueError, IndexError):
pass
return None
async def claim_attachment(attachment_id: str):
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if att is None:
print(
f"[Worker] Файл {attachment_id} не найден",
flush=True,
)
return None
if att.status == "ready" and att.thumbnail_key:
print(
f"[Worker] Файл {attachment_id} уже обработан",
flush=True,
)
return None
if att.status == "failed":
print(
f"[Worker] Файл {attachment_id} уже failed",
flush=True,
)
return None
now = datetime.now(timezone.utc)
if (
att.status == "processing"
and att.lease_expires_at is not None
and att.lease_expires_at > now
):
raise LeaseBusyError()
lease_expires_at = now + timedelta(
seconds=LEASE_SECONDS
)
att.status = "processing"
att.error_code = None
att.lease_expires_at = lease_expires_at
object_key = att.object_key
return object_key, lease_expires_at
async def release_lease(
attachment_id: str,
lease_expires_at: datetime,
):
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if (
att is not None
and att.status == "processing"
and att.lease_expires_at == lease_expires_at
):
att.status = "queued"
att.lease_expires_at = None
async def mark_failed(
attachment_id: str,
error_code: str,
):
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if att is None or att.status == "ready":
return
if att.status == "processing":
raise LeaseBusyError()
att.status = "failed"
att.error_code = error_code
att.lease_expires_at = None
print(
f"[Worker] Файл {attachment_id}: "
f"failed, error_code={error_code}",
flush=True,
)
async def process_image(image_data: bytes):
with tempfile.TemporaryDirectory() as temp_dir:
directory = Path(temp_dir)
input_path = directory / "original"
input_path.write_bytes(image_data)
try:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"app.image_processor",
str(input_path),
str(directory),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
except OSError as exc:
raise RetryableProcessingError(
"image_processor_start_error"
) from exc
try:
await asyncio.wait_for(
process.communicate(),
timeout=PROCESS_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError as exc:
if process.returncode is None:
try:
process.kill()
except ProcessLookupError:
pass
await process.communicate()
raise RetryableProcessingError(
"image_processing_timeout"
) from exc
if process.returncode == 20:
raise PermanentProcessingError(
"image_too_many_pixels"
)
if process.returncode == 21:
raise PermanentProcessingError(
"image_processing_error"
)
if process.returncode != 0:
raise RetryableProcessingError(
"image_processor_error"
)
try:
thumb_bytes = (
directory / "thumbnail.jpg"
).read_bytes()
crop_bytes = (
directory / "crop.jpg"
).read_bytes()
webp_bytes = (
directory / "optimized.webp"
).read_bytes()
with Image.open(
io.BytesIO(thumb_bytes)
) as thumbnail:
thumb_width, thumb_height = thumbnail.size
except OSError as exc:
raise RetryableProcessingError(
"image_processor_output_error"
) from exc
return (
thumb_bytes,
crop_bytes,
webp_bytes,
thumb_width,
thumb_height,
)
async def process_attachment(attachment_id: str):
print(
f"[Worker] Обработка файла: {attachment_id}",
flush=True,
)
claim = await claim_attachment(attachment_id)
if claim is None:
return
object_key, lease_expires_at = claim
try:
try:
response = minio_client.get_object(
S3_BUCKET,
object_key,
)
try:
image_data = response.read(
MAX_IMAGE_BYTES + 1
)
finally:
response.close()
response.release_conn()
except Exception as exc:
raise RetryableProcessingError(
"s3_download_error"
) from exc
if len(image_data) > MAX_IMAGE_BYTES:
raise PermanentProcessingError(
"image_too_large"
)
(
thumb_bytes,
crop_bytes,
webp_bytes,
thumb_width,
thumb_height,
) = await process_image(image_data)
thumb_key = f"thumbnails/{object_key}"
crop_key = f"crops/{attachment_id}.jpg"
webp_key = f"webp/{attachment_id}.webp"
try:
minio_client.put_object(
S3_BUCKET,
thumb_key,
io.BytesIO(thumb_bytes),
len(thumb_bytes),
content_type="image/jpeg",
)
minio_client.put_object(
S3_BUCKET,
crop_key,
io.BytesIO(crop_bytes),
len(crop_bytes),
content_type="image/jpeg",
)
minio_client.put_object(
S3_BUCKET,
webp_key,
io.BytesIO(webp_bytes),
len(webp_bytes),
content_type="image/webp",
)
except Exception as exc:
raise RetryableProcessingError(
"s3_upload_error"
) from exc
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if (
att is None
or att.status != "processing"
or att.lease_expires_at != lease_expires_at
):
raise LeaseLostError()
att.thumbnail_key = thumb_key
att.thumbnail_sha256 = hashlib.sha256(
thumb_bytes
).hexdigest()
att.thumbnail_size_bytes = len(thumb_bytes)
att.thumbnail_width = thumb_width
att.thumbnail_height = thumb_height
att.crop_key = crop_key
att.webp_key = webp_key
att.status = "ready"
att.error_code = None
att.lease_expires_at = None
print(
f"[Worker] Успешно обработан {attachment_id}",
flush=True,
)
except Exception:
await release_lease(
attachment_id,
lease_expires_at,
)
raise
async def main():
connection = await aio_pika.connect_robust(
RABBITMQ_URL
)
async with connection:
channel = await connection.channel(
publisher_confirms=True
)
await channel.set_qos(
prefetch_count=PREFETCH
)
queue = await channel.declare_queue(
QUEUE_NAME,
durable=True,
)
semaphore = asyncio.Semaphore(CONCURRENCY)
print(
f"[Worker] CONCURRENCY={CONCURRENCY}, "
f"PREFETCH={PREFETCH}",
flush=True,
)
async def on_message(
message: aio_pika.IncomingMessage
):
async with semaphore:
started = time.perf_counter()
try:
data = json.loads(
message.body.decode()
)
attachment_id = str(
uuid.UUID(data["attachment_id"])
)
except (
ValueError,
KeyError,
TypeError,
):
print(
"[Worker] Некорректное сообщение",
flush=True,
)
await message.reject(
requeue=False
)
return
headers = dict(message.headers or {})
try:
attempt = max(
1,
int(headers.get("x-attempt", 1)),
)
except (ValueError, TypeError):
attempt = 1
print(
f"[Worker] Попытка "
f"{attempt}/{MAX_ATTEMPTS} "
f"для {attachment_id}",
flush=True,
)
try:
await process_attachment(
attachment_id
)
except LeaseBusyError:
print(
"[Worker] Задание уже "
"обрабатывается, ждём Lease",
flush=True,
)
await asyncio.sleep(5)
await message.nack(
requeue=True
)
return
except LeaseLostError:
print(
"[Worker] Lease перешёл "
"другому Worker",
flush=True,
)
await message.ack()
return
except PermanentProcessingError as exc:
try:
await mark_failed(
attachment_id,
exc.error_code,
)
except Exception:
await message.nack(
requeue=True
)
return
await message.ack()
return
except Exception as exc:
error_code = (
exc.error_code
if isinstance(
exc,
RetryableProcessingError,
)
else "worker_error"
)
if attempt < MAX_ATTEMPTS:
next_attempt = attempt + 1
print(
f"[Worker] Ошибка {error_code}. "
f"Повтор "
f"{next_attempt}/{MAX_ATTEMPTS}",
flush=True,
)
headers["x-attempt"] = next_attempt
base_delay = min(
30,
2 ** (attempt - 1),
)
delay = base_delay + random.uniform(
0,
base_delay / 2,
)
print(
f"[Worker] Повтор через "
f"{delay:.1f} сек.",
flush=True,
)
await asyncio.sleep(delay)
try:
await channel.default_exchange.publish(
aio_pika.Message(
body=message.body,
headers=headers,
delivery_mode=(
aio_pika.DeliveryMode.PERSISTENT
),
content_type="application/json",
),
routing_key=QUEUE_NAME,
)
except Exception:
await message.nack(
requeue=True
)
return
await message.ack()
return
print(
f"[Worker] Попытки исчерпаны: "
f"{attachment_id}",
flush=True,
)
try:
await mark_failed(
attachment_id,
error_code,
)
except Exception:
await message.nack(
requeue=True
)
return
await message.ack()
return
await message.ack()
duration = time.perf_counter() - started
rss_mb = get_rss_mb()
print(
f"[Metrics] attachment_id={attachment_id} "
f"duration_s={duration:.3f} "
f"rss_mb={rss_mb}",
flush=True,
)
print(
"[Worker] Ждем сообщения из RabbitMQ...",
flush=True,
)
await queue.consume(on_message)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())