feat: complete lab5 with async processing, worker, variants and resource limits

This commit is contained in:
2026-11-11 21:50:58 +03:00
parent 4df4307367
commit 2d18329ed1
2 changed files with 95 additions and 90 deletions
+85 -90
View File
@@ -8,14 +8,12 @@ from datetime import datetime, timezone
import pika
from PIL import Image, ImageOps
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
from sqlalchemy.pool import NullPool
from .db.models import Attachment
from .s3_client import s3, BUCKET
DATABASE_URL = os.getenv("DATABASE_URL")
engine = create_async_engine(DATABASE_URL)
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False)
RABBITMQ_URL = os.getenv("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/")
QUEUE_NAME = "attachment_tasks"
@@ -23,100 +21,97 @@ async def process_attachment(task: dict):
att_id = task["attachment_id"]
s3_key = task["s3_key"]
async with AsyncSessionLocal() as session:
att = await session.get(Attachment, att_id)
if not att:
return
# ВАЖНО: создаем engine внутри функции с NullPool, чтобы избежать ошибки разных event loop
engine = create_async_engine(DATABASE_URL, poolclass=NullPool)
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False)
# идемпотентность: если уже обработано, пропускаем
if att.status == "ready" and len(att.variants) == 3:
return
try:
async with AsyncSessionLocal() as session:
att = await session.get(Attachment, att_id)
if not att:
return
try:
# получаем оригинал
obj = s3.get_object(Bucket=BUCKET, Key=s3_key)
content = obj["Body"].read()
img = Image.open(io.BytesIO(content))
# убираем прозрачность (композируем на белый) и разворачиваем по exif
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA"):
background = Image.new("RGB", img.size, (255, 255, 255))
background.paste(img, mask=img.split()[-1])
img = background
elif img.mode != "RGB":
img = img.convert("RGB")
# идемпотентность: если уже обработано, пропускаем
if att.status == "ready" and len(att.variants or []) == 3:
return
variants_meta = []
try:
# получаем оригинал
obj = s3.get_object(Bucket=BUCKET, Key=s3_key)
content = obj["Body"].read()
img = Image.open(io.BytesIO(content))
# разворачиваем по exif и убираем прозрачность
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA"):
background = Image.new("RGB", img.size, (255, 255, 255))
background.paste(img, mask=img.split()[-1])
img = background
elif img.mode != "RGB":
img = img.convert("RGB")
# --- 1. Thumbnail (256x256, вписанный) ---
thumb = img.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumb_bytes = io.BytesIO()
thumb.save(thumb_bytes, format="JPEG")
thumb_data = thumb_bytes.getvalue()
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_thumb.jpg", Body=thumb_data, ContentType="image/jpeg")
variants_meta.append({
"name": "thumbnail",
"content_type": "image/jpeg",
"width": thumb.width,
"height": thumb.height,
"size_bytes": len(thumb_data),
"sha256": hashlib.sha256(thumb_data).hexdigest()
})
variants_meta = []
# --- 2. Crop (128x128, центрированный квадрат) ---
crop_img = img.copy()
width, height = crop_img.size
min_dim = min(width, height)
left = (width - min_dim) / 2
top = (height - min_dim) / 2
right = (width + min_dim) / 2
bottom = (height + min_dim) / 2
crop_img = crop_img.crop((left, top, right, bottom))
crop_img = crop_img.resize((128, 128), Image.Resampling.LANCZOS)
crop_bytes = io.BytesIO()
crop_img.save(crop_bytes, format="JPEG")
crop_data = crop_bytes.getvalue()
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_crop.jpg", Body=crop_data, ContentType="image/jpeg")
variants_meta.append({
"name": "crop",
"content_type": "image/jpeg",
"width": 128,
"height": 128,
"size_bytes": len(crop_data),
"sha256": hashlib.sha256(crop_data).hexdigest()
})
# 1. Thumbnail (256x256)
thumb = img.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumb_bytes = io.BytesIO()
thumb.save(thumb_bytes, format="JPEG")
thumb_data = thumb_bytes.getvalue()
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_thumb.jpg", Body=thumb_data, ContentType="image/jpeg")
variants_meta.append({
"name": "thumbnail", "content_type": "image/jpeg",
"width": thumb.width, "height": thumb.height,
"size_bytes": len(thumb_data), "sha256": hashlib.sha256(thumb_data).hexdigest()
})
# --- 3. Optimized (WebP, до 1280x1280 без увеличения) ---
opt_img = img.copy()
opt_img.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
opt_bytes = io.BytesIO()
opt_img.save(opt_bytes, format="WEBP", quality=85)
opt_data = opt_bytes.getvalue()
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_opt.webp", Body=opt_data, ContentType="image/webp")
variants_meta.append({
"name": "optimized",
"content_type": "image/webp",
"width": opt_img.width,
"height": opt_img.height,
"size_bytes": len(opt_data),
"sha256": hashlib.sha256(opt_data).hexdigest()
})
# 2. Crop (128x128)
crop_img = img.copy()
width, height = crop_img.size
min_dim = min(width, height)
left = (width - min_dim) / 2
top = (height - min_dim) / 2
right = (width + min_dim) / 2
bottom = (height + min_dim) / 2
crop_img = crop_img.crop((left, top, right, bottom))
crop_img = crop_img.resize((128, 128), Image.Resampling.LANCZOS)
crop_bytes = io.BytesIO()
crop_img.save(crop_bytes, format="JPEG")
crop_data = crop_bytes.getvalue()
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_crop.jpg", Body=crop_data, ContentType="image/jpeg")
variants_meta.append({
"name": "crop", "content_type": "image/jpeg",
"width": 128, "height": 128,
"size_bytes": len(crop_data), "sha256": hashlib.sha256(crop_data).hexdigest()
})
# обновляем БД
att.status = "ready"
att.error_code = None
att.variants = variants_meta
att.updated_at = datetime.now(timezone.utc)
await session.commit()
# 3. Optimized (WebP)
opt_img = img.copy()
opt_img.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
opt_bytes = io.BytesIO()
opt_img.save(opt_bytes, format="WEBP", quality=85)
opt_data = opt_bytes.getvalue()
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_opt.webp", Body=opt_data, ContentType="image/webp")
variants_meta.append({
"name": "optimized", "content_type": "image/webp",
"width": opt_img.width, "height": opt_img.height,
"size_bytes": len(opt_data), "sha256": hashlib.sha256(opt_data).hexdigest()
})
except Exception as e:
att.status = "failed"
att.error_code = "processing_failed"
att.updated_at = datetime.now(timezone.utc)
await session.commit()
print(f"Error processing {att_id}: {e}")
att.status = "ready"
att.error_code = None
att.variants = variants_meta
att.updated_at = datetime.now(timezone.utc)
await session.commit()
except Exception as e:
att.status = "failed"
att.error_code = "processing_failed"
att.updated_at = datetime.now(timezone.utc)
await session.commit()
print(f"Error processing {att_id}: {e}", flush=True)
finally:
await engine.dispose()
def callback(ch, method, properties, body):
task = json.loads(body)
@@ -129,7 +124,7 @@ def main():
channel.queue_declare(queue=QUEUE_NAME, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=callback)
print("Worker started, waiting for messages...")
print("Worker started, waiting for messages...", flush=True)
channel.start_consuming()
if __name__ == "__main__":
+10
View File
@@ -88,6 +88,11 @@ services:
condition: service_healthy
migrator:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
worker:
build: .
@@ -108,6 +113,11 @@ services:
condition: service_healthy
migrator:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: '1.0'
memory: 1G
nginx:
image: nginx:1.25-alpine