From 2d18329ed11d2d8dac7f9164ba4281066cc0f7e0 Mon Sep 17 00:00:00 2001 From: Maksimova Snezhana Date: Wed, 11 Nov 2026 21:50:58 +0300 Subject: [PATCH] feat: complete lab5 with async processing, worker, variants and resource limits --- app/worker.py | 175 ++++++++++++++++++++++++-------------------------- compose.yaml | 10 +++ 2 files changed, 95 insertions(+), 90 deletions(-) diff --git a/app/worker.py b/app/worker.py index 24703ed..1b048c3 100644 --- a/app/worker.py +++ b/app/worker.py @@ -8,14 +8,12 @@ from datetime import datetime, timezone import pika from PIL import Image, ImageOps from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +from sqlalchemy.pool import NullPool from .db.models import Attachment from .s3_client import s3, BUCKET DATABASE_URL = os.getenv("DATABASE_URL") -engine = create_async_engine(DATABASE_URL) -AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False) - RABBITMQ_URL = os.getenv("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/") QUEUE_NAME = "attachment_tasks" @@ -23,100 +21,97 @@ async def process_attachment(task: dict): att_id = task["attachment_id"] s3_key = task["s3_key"] - async with AsyncSessionLocal() as session: - att = await session.get(Attachment, att_id) - if not att: - return + # ВАЖНО: создаем engine внутри функции с NullPool, чтобы избежать ошибки разных event loop + engine = create_async_engine(DATABASE_URL, poolclass=NullPool) + AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False) - # идемпотентность: если уже обработано, пропускаем - if att.status == "ready" and len(att.variants) == 3: - return + try: + async with AsyncSessionLocal() as session: + att = await session.get(Attachment, att_id) + if not att: + return - try: - # получаем оригинал - obj = s3.get_object(Bucket=BUCKET, Key=s3_key) - content = obj["Body"].read() - img = Image.open(io.BytesIO(content)) - - # убираем прозрачность (композируем на белый) и разворачиваем по exif - img = ImageOps.exif_transpose(img) - if img.mode in ("RGBA", "LA"): - background = Image.new("RGB", img.size, (255, 255, 255)) - background.paste(img, mask=img.split()[-1]) - img = background - elif img.mode != "RGB": - img = img.convert("RGB") + # идемпотентность: если уже обработано, пропускаем + if att.status == "ready" and len(att.variants or []) == 3: + return - variants_meta = [] + try: + # получаем оригинал + obj = s3.get_object(Bucket=BUCKET, Key=s3_key) + content = obj["Body"].read() + img = Image.open(io.BytesIO(content)) + + # разворачиваем по exif и убираем прозрачность + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA"): + background = Image.new("RGB", img.size, (255, 255, 255)) + background.paste(img, mask=img.split()[-1]) + img = background + elif img.mode != "RGB": + img = img.convert("RGB") - # --- 1. Thumbnail (256x256, вписанный) --- - thumb = img.copy() - thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) - thumb_bytes = io.BytesIO() - thumb.save(thumb_bytes, format="JPEG") - thumb_data = thumb_bytes.getvalue() - s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_thumb.jpg", Body=thumb_data, ContentType="image/jpeg") - variants_meta.append({ - "name": "thumbnail", - "content_type": "image/jpeg", - "width": thumb.width, - "height": thumb.height, - "size_bytes": len(thumb_data), - "sha256": hashlib.sha256(thumb_data).hexdigest() - }) + variants_meta = [] - # --- 2. Crop (128x128, центрированный квадрат) --- - crop_img = img.copy() - width, height = crop_img.size - min_dim = min(width, height) - left = (width - min_dim) / 2 - top = (height - min_dim) / 2 - right = (width + min_dim) / 2 - bottom = (height + min_dim) / 2 - crop_img = crop_img.crop((left, top, right, bottom)) - crop_img = crop_img.resize((128, 128), Image.Resampling.LANCZOS) - crop_bytes = io.BytesIO() - crop_img.save(crop_bytes, format="JPEG") - crop_data = crop_bytes.getvalue() - s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_crop.jpg", Body=crop_data, ContentType="image/jpeg") - variants_meta.append({ - "name": "crop", - "content_type": "image/jpeg", - "width": 128, - "height": 128, - "size_bytes": len(crop_data), - "sha256": hashlib.sha256(crop_data).hexdigest() - }) + # 1. Thumbnail (256x256) + thumb = img.copy() + thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) + thumb_bytes = io.BytesIO() + thumb.save(thumb_bytes, format="JPEG") + thumb_data = thumb_bytes.getvalue() + s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_thumb.jpg", Body=thumb_data, ContentType="image/jpeg") + variants_meta.append({ + "name": "thumbnail", "content_type": "image/jpeg", + "width": thumb.width, "height": thumb.height, + "size_bytes": len(thumb_data), "sha256": hashlib.sha256(thumb_data).hexdigest() + }) - # --- 3. Optimized (WebP, до 1280x1280 без увеличения) --- - opt_img = img.copy() - opt_img.thumbnail((1280, 1280), Image.Resampling.LANCZOS) - opt_bytes = io.BytesIO() - opt_img.save(opt_bytes, format="WEBP", quality=85) - opt_data = opt_bytes.getvalue() - s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_opt.webp", Body=opt_data, ContentType="image/webp") - variants_meta.append({ - "name": "optimized", - "content_type": "image/webp", - "width": opt_img.width, - "height": opt_img.height, - "size_bytes": len(opt_data), - "sha256": hashlib.sha256(opt_data).hexdigest() - }) + # 2. Crop (128x128) + crop_img = img.copy() + width, height = crop_img.size + min_dim = min(width, height) + left = (width - min_dim) / 2 + top = (height - min_dim) / 2 + right = (width + min_dim) / 2 + bottom = (height + min_dim) / 2 + crop_img = crop_img.crop((left, top, right, bottom)) + crop_img = crop_img.resize((128, 128), Image.Resampling.LANCZOS) + crop_bytes = io.BytesIO() + crop_img.save(crop_bytes, format="JPEG") + crop_data = crop_bytes.getvalue() + s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_crop.jpg", Body=crop_data, ContentType="image/jpeg") + variants_meta.append({ + "name": "crop", "content_type": "image/jpeg", + "width": 128, "height": 128, + "size_bytes": len(crop_data), "sha256": hashlib.sha256(crop_data).hexdigest() + }) - # обновляем БД - att.status = "ready" - att.error_code = None - att.variants = variants_meta - att.updated_at = datetime.now(timezone.utc) - await session.commit() + # 3. Optimized (WebP) + opt_img = img.copy() + opt_img.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + opt_bytes = io.BytesIO() + opt_img.save(opt_bytes, format="WEBP", quality=85) + opt_data = opt_bytes.getvalue() + s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_opt.webp", Body=opt_data, ContentType="image/webp") + variants_meta.append({ + "name": "optimized", "content_type": "image/webp", + "width": opt_img.width, "height": opt_img.height, + "size_bytes": len(opt_data), "sha256": hashlib.sha256(opt_data).hexdigest() + }) - except Exception as e: - att.status = "failed" - att.error_code = "processing_failed" - att.updated_at = datetime.now(timezone.utc) - await session.commit() - print(f"Error processing {att_id}: {e}") + att.status = "ready" + att.error_code = None + att.variants = variants_meta + att.updated_at = datetime.now(timezone.utc) + await session.commit() + + except Exception as e: + att.status = "failed" + att.error_code = "processing_failed" + att.updated_at = datetime.now(timezone.utc) + await session.commit() + print(f"Error processing {att_id}: {e}", flush=True) + finally: + await engine.dispose() def callback(ch, method, properties, body): task = json.loads(body) @@ -129,7 +124,7 @@ def main(): channel.queue_declare(queue=QUEUE_NAME, durable=True) channel.basic_qos(prefetch_count=1) channel.basic_consume(queue=QUEUE_NAME, on_message_callback=callback) - print("Worker started, waiting for messages...") + print("Worker started, waiting for messages...", flush=True) channel.start_consuming() if __name__ == "__main__": diff --git a/compose.yaml b/compose.yaml index 3771447..a43c9e9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -88,6 +88,11 @@ services: condition: service_healthy migrator: condition: service_completed_successfully + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M worker: build: . @@ -108,6 +113,11 @@ services: condition: service_healthy migrator: condition: service_completed_successfully + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G nginx: image: nginx:1.25-alpine