import asyncio import hashlib import io import json import os import random import sys import tempfile import time import uuid from datetime import datetime, timedelta, timezone from pathlib import Path import aio_pika from minio import Minio from PIL import Image from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker from app.models.attachment import Attachment from app.models.user import User from app.models.chat import Chat, ChatMember from app.config import ( DATABASE_URL, S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET, ) RABBITMQ_URL = os.getenv( "RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/", ) QUEUE_NAME = "image_tasks" MAX_ATTEMPTS = max( 1, int(os.getenv("WORKER_MAX_ATTEMPTS", "5")) ) CONCURRENCY = max( 1, int(os.getenv("WORKER_CONCURRENCY", "1")) ) PREFETCH = max( 1, int(os.getenv("WORKER_PREFETCH", "1")) ) MAX_IMAGE_BYTES = 10 * 1024 * 1024 PROCESS_TIMEOUT_SECONDS = 30 LEASE_SECONDS = 300 endpoint = S3_ENDPOINT.replace( "http://", "" ).replace( "https://", "" ) minio_client = Minio( endpoint, access_key=S3_ACCESS_KEY_ID, secret_key=S3_SECRET_ACCESS_KEY, secure=False, ) engine = create_async_engine(DATABASE_URL) async_session_maker = async_sessionmaker( engine, expire_on_commit=False, ) class RetryableProcessingError(Exception): def __init__(self, error_code: str): self.error_code = error_code super().__init__(error_code) class PermanentProcessingError(Exception): def __init__(self, error_code: str): self.error_code = error_code super().__init__(error_code) class LeaseBusyError(Exception): pass class LeaseLostError(Exception): pass def get_rss_mb(): try: with open("/proc/self/status") as file: for line in file: if line.startswith("VmRSS:"): kb = int(line.split()[1]) return round(kb / 1024, 2) except (OSError, ValueError, IndexError): pass return None async def claim_attachment(attachment_id: str): async with async_session_maker() as session: async with session.begin(): att = await session.get( Attachment, uuid.UUID(attachment_id), with_for_update=True, ) if att is None: print( f"[Worker] Файл {attachment_id} не найден", flush=True, ) return None if att.status == "ready" and att.thumbnail_key: print( f"[Worker] Файл {attachment_id} уже обработан", flush=True, ) return None if att.status == "failed": print( f"[Worker] Файл {attachment_id} уже failed", flush=True, ) return None now = datetime.now(timezone.utc) if ( att.status == "processing" and att.lease_expires_at is not None and att.lease_expires_at > now ): raise LeaseBusyError() lease_expires_at = now + timedelta( seconds=LEASE_SECONDS ) att.status = "processing" att.error_code = None att.lease_expires_at = lease_expires_at object_key = att.object_key return object_key, lease_expires_at async def release_lease( attachment_id: str, lease_expires_at: datetime, ): async with async_session_maker() as session: async with session.begin(): att = await session.get( Attachment, uuid.UUID(attachment_id), with_for_update=True, ) if ( att is not None and att.status == "processing" and att.lease_expires_at == lease_expires_at ): att.status = "queued" att.lease_expires_at = None async def mark_failed( attachment_id: str, error_code: str, ): async with async_session_maker() as session: async with session.begin(): att = await session.get( Attachment, uuid.UUID(attachment_id), with_for_update=True, ) if att is None or att.status == "ready": return if att.status == "processing": raise LeaseBusyError() att.status = "failed" att.error_code = error_code att.lease_expires_at = None print( f"[Worker] Файл {attachment_id}: " f"failed, error_code={error_code}", flush=True, ) async def process_image(image_data: bytes): with tempfile.TemporaryDirectory() as temp_dir: directory = Path(temp_dir) input_path = directory / "original" input_path.write_bytes(image_data) try: process = await asyncio.create_subprocess_exec( sys.executable, "-m", "app.image_processor", str(input_path), str(directory), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) except OSError as exc: raise RetryableProcessingError( "image_processor_start_error" ) from exc try: await asyncio.wait_for( process.communicate(), timeout=PROCESS_TIMEOUT_SECONDS, ) except asyncio.TimeoutError as exc: if process.returncode is None: try: process.kill() except ProcessLookupError: pass await process.communicate() raise RetryableProcessingError( "image_processing_timeout" ) from exc if process.returncode == 20: raise PermanentProcessingError( "image_too_many_pixels" ) if process.returncode == 21: raise PermanentProcessingError( "image_processing_error" ) if process.returncode != 0: raise RetryableProcessingError( "image_processor_error" ) try: thumb_bytes = ( directory / "thumbnail.jpg" ).read_bytes() crop_bytes = ( directory / "crop.jpg" ).read_bytes() webp_bytes = ( directory / "optimized.webp" ).read_bytes() with Image.open( io.BytesIO(thumb_bytes) ) as thumbnail: thumb_width, thumb_height = thumbnail.size except OSError as exc: raise RetryableProcessingError( "image_processor_output_error" ) from exc return ( thumb_bytes, crop_bytes, webp_bytes, thumb_width, thumb_height, ) async def process_attachment(attachment_id: str): print( f"[Worker] Обработка файла: {attachment_id}", flush=True, ) claim = await claim_attachment(attachment_id) if claim is None: return object_key, lease_expires_at = claim try: try: response = minio_client.get_object( S3_BUCKET, object_key, ) try: image_data = response.read( MAX_IMAGE_BYTES + 1 ) finally: response.close() response.release_conn() except Exception as exc: raise RetryableProcessingError( "s3_download_error" ) from exc if len(image_data) > MAX_IMAGE_BYTES: raise PermanentProcessingError( "image_too_large" ) ( thumb_bytes, crop_bytes, webp_bytes, thumb_width, thumb_height, ) = await process_image(image_data) thumb_key = f"thumbnails/{object_key}" crop_key = f"crops/{attachment_id}.jpg" webp_key = f"webp/{attachment_id}.webp" try: minio_client.put_object( S3_BUCKET, thumb_key, io.BytesIO(thumb_bytes), len(thumb_bytes), content_type="image/jpeg", ) minio_client.put_object( S3_BUCKET, crop_key, io.BytesIO(crop_bytes), len(crop_bytes), content_type="image/jpeg", ) minio_client.put_object( S3_BUCKET, webp_key, io.BytesIO(webp_bytes), len(webp_bytes), content_type="image/webp", ) except Exception as exc: raise RetryableProcessingError( "s3_upload_error" ) from exc async with async_session_maker() as session: async with session.begin(): att = await session.get( Attachment, uuid.UUID(attachment_id), with_for_update=True, ) if ( att is None or att.status != "processing" or att.lease_expires_at != lease_expires_at ): raise LeaseLostError() att.thumbnail_key = thumb_key att.thumbnail_sha256 = hashlib.sha256( thumb_bytes ).hexdigest() att.thumbnail_size_bytes = len(thumb_bytes) att.thumbnail_width = thumb_width att.thumbnail_height = thumb_height att.crop_key = crop_key att.webp_key = webp_key att.status = "ready" att.error_code = None att.lease_expires_at = None print( f"[Worker] Успешно обработан {attachment_id}", flush=True, ) except Exception: await release_lease( attachment_id, lease_expires_at, ) raise async def main(): connection = await aio_pika.connect_robust( RABBITMQ_URL ) async with connection: channel = await connection.channel( publisher_confirms=True ) await channel.set_qos( prefetch_count=PREFETCH ) queue = await channel.declare_queue( QUEUE_NAME, durable=True, ) semaphore = asyncio.Semaphore(CONCURRENCY) print( f"[Worker] CONCURRENCY={CONCURRENCY}, " f"PREFETCH={PREFETCH}", flush=True, ) async def on_message( message: aio_pika.IncomingMessage ): async with semaphore: started = time.perf_counter() try: data = json.loads( message.body.decode() ) attachment_id = str( uuid.UUID(data["attachment_id"]) ) except ( ValueError, KeyError, TypeError, ): print( "[Worker] Некорректное сообщение", flush=True, ) await message.reject( requeue=False ) return headers = dict(message.headers or {}) try: attempt = max( 1, int(headers.get("x-attempt", 1)), ) except (ValueError, TypeError): attempt = 1 print( f"[Worker] Попытка " f"{attempt}/{MAX_ATTEMPTS} " f"для {attachment_id}", flush=True, ) try: await process_attachment( attachment_id ) except LeaseBusyError: print( "[Worker] Задание уже " "обрабатывается, ждём Lease", flush=True, ) await asyncio.sleep(5) await message.nack( requeue=True ) return except LeaseLostError: print( "[Worker] Lease перешёл " "другому Worker", flush=True, ) await message.ack() return except PermanentProcessingError as exc: try: await mark_failed( attachment_id, exc.error_code, ) except Exception: await message.nack( requeue=True ) return await message.ack() return except Exception as exc: error_code = ( exc.error_code if isinstance( exc, RetryableProcessingError, ) else "worker_error" ) if attempt < MAX_ATTEMPTS: next_attempt = attempt + 1 print( f"[Worker] Ошибка {error_code}. " f"Повтор " f"{next_attempt}/{MAX_ATTEMPTS}", flush=True, ) headers["x-attempt"] = next_attempt base_delay = min( 30, 2 ** (attempt - 1), ) delay = base_delay + random.uniform( 0, base_delay / 2, ) print( f"[Worker] Повтор через " f"{delay:.1f} сек.", flush=True, ) await asyncio.sleep(delay) try: await channel.default_exchange.publish( aio_pika.Message( body=message.body, headers=headers, delivery_mode=( aio_pika.DeliveryMode.PERSISTENT ), content_type="application/json", ), routing_key=QUEUE_NAME, ) except Exception: await message.nack( requeue=True ) return await message.ack() return print( f"[Worker] Попытки исчерпаны: " f"{attachment_id}", flush=True, ) try: await mark_failed( attachment_id, error_code, ) except Exception: await message.nack( requeue=True ) return await message.ack() return await message.ack() duration = time.perf_counter() - started rss_mb = get_rss_mb() print( f"[Metrics] attachment_id={attachment_id} " f"duration_s={duration:.3f} " f"rss_mb={rss_mb}", flush=True, ) print( "[Worker] Ждем сообщения из RabbitMQ...", flush=True, ) await queue.consume(on_message) await asyncio.Future() if __name__ == "__main__": asyncio.run(main())