import os import sys import uuid import hashlib import asyncio import logging import tempfile import json from datetime import datetime, timezone, timedelta from sqlalchemy import select, update from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession from sqlalchemy.orm import sessionmaker import aio_pika from minio import Minio from PIL import Image import io logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') logger = logging.getLogger(__name__) DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db") BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "") S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin") S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin") S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments") WORKER_ID = str(uuid.uuid4())[:8] JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30")) JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) engine = create_async_engine(DATABASE_URL, echo=False) AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False) async def process_image(body: bytes, att_id: str, chat_id: str): img = Image.open(io.BytesIO(body)).convert("RGB") width, height = img.size variants_to_save = [] # 1. Thumbnail (вписан в 256x256) thumb = img.copy() thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) thumb_bytes = io.BytesIO() thumb.save(thumb_bytes, format="JPEG", quality=85) thumb_data = thumb_bytes.getvalue() thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg" with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp: tmp.write(thumb_data) s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg") os.unlink(tmp.name) variants_to_save.append({ "variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg", "size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height, "sha256": hashlib.sha256(thumb_data).hexdigest() }) # 2. Crop (центральный квадрат 128x128) min_dim = min(width, height) left = (width - min_dim) // 2 top = (height - min_dim) // 2 crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS) crop_bytes = io.BytesIO() crop.save(crop_bytes, format="JPEG", quality=85) crop_data = crop_bytes.getvalue() crop_key = f"variants/{chat_id}/{att_id}/crop.jpg" with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp: tmp.write(crop_data) s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg") os.unlink(tmp.name) variants_to_save.append({ "variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg", "size_bytes": len(crop_data), "width": 128, "height": 128, "sha256": hashlib.sha256(crop_data).hexdigest() }) # 3. Optimized WebP (вписан в 1280x1280) opt = img.copy() opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS) opt_bytes = io.BytesIO() opt.save(opt_bytes, format="WEBP", quality=80) opt_data = opt_bytes.getvalue() opt_key = f"variants/{chat_id}/{att_id}/optimized.webp" with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp: tmp.write(opt_data) s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp") os.unlink(tmp.name) variants_to_save.append({ "variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp", "size_bytes": len(opt_data), "width": opt.width, "height": opt.height, "sha256": hashlib.sha256(opt_data).hexdigest() }) # 4. Сохраняем оригинал в конечную папку ext = "png" if b"png" in body[:8] else "jpeg" final_key = f"attachments/{chat_id}/{att_id}.{ext}" content_type = "image/png" if ext == "png" else "image/jpeg" with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp: tmp.write(body) s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type) os.unlink(tmp.name) return final_key, variants_to_save async def process_message(message: aio_pika.IncomingMessage): async with message.process(requeue=False): try: payload = message.body.decode() data = json.loads(payload) job_id = data["job_id"] att_id = data["attachment_id"] attempt = data["attempt"] logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}") async with AsyncSessionLocal() as db: from app.models import AttachmentJob, Attachment, AttachmentVariant # 1. Блокируем задачу для обработки (Claim) stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update() result = await db.execute(stmt) job = result.scalar_one_or_none() if not job: logger.warning(f"Job {job_id} not found") return if job.status in ("ready", "failed"): logger.info(f"Job {job_id} already {job.status}, skipping") return if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc): logger.info(f"Job {job_id} is already being processed by another worker") return now = datetime.now(timezone.utc) job.status = "processing" job.attempt = attempt job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS) job.lease_owner = WORKER_ID await db.commit() # 2. Получаем метаданные вложения att_stmt = select(Attachment).where(Attachment.id == att_id) att_result = await db.execute(att_stmt) attachment = att_result.scalar_one_or_none() if not attachment or not attachment.staging_key: job.status = "failed" job.error_code = "missing_staging" await db.commit() return staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления # 3. Скачиваем из staging response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete) body = response.read() response.close() response.release_conn() # 4. Обрабатываем изображение final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id)) # 5. Фиксируем результат в БД attachment.status = "ready" attachment.s3_key = final_key attachment.staging_key = None # Очищаем staging job.status = "ready" for v_data in variants_data: variant = AttachmentVariant( attachment_id=attachment.id, **v_data ) db.add(variant) await db.commit() logger.info(f"Job {job_id} completed successfully") # 6. Удаляем staging из S3 s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete) except Exception as e: logger.error(f"Error processing message: {e}") try: async with AsyncSessionLocal() as db: from app.models import AttachmentJob stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update() result = await db.execute(stmt) job = result.scalar_one_or_none() if job: if job.attempt >= JOB_MAX_ATTEMPTS: job.status = "failed" job.error_code = "retry_exhausted" else: job.status = "queued" job.attempt += 1 job.lease_until = None await db.commit() except Exception as db_err: logger.error(f"Failed to update job status after error: {db_err}") async def main(): logger.info("Starting worker...") connection = await aio_pika.connect_robust(BROKER_URL) channel = await connection.channel() queue = await channel.declare_queue("attachment.process.v1", durable=True) logger.info("Waiting for messages...") await queue.consume(process_message) await asyncio.Future() if __name__ == "__main__": asyncio.run(main())