forked from mtusi2026/mtusi2026lab5
feat: complete lab5 with async processing, worker, variants and resource limits
This commit is contained in:
+85
-90
@@ -8,14 +8,12 @@ from datetime import datetime, timezone
|
||||
import pika
|
||||
from PIL import Image, ImageOps
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from .db.models import Attachment
|
||||
from .s3_client import s3, BUCKET
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL")
|
||||
engine = create_async_engine(DATABASE_URL)
|
||||
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
RABBITMQ_URL = os.getenv("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/")
|
||||
QUEUE_NAME = "attachment_tasks"
|
||||
|
||||
@@ -23,100 +21,97 @@ async def process_attachment(task: dict):
|
||||
att_id = task["attachment_id"]
|
||||
s3_key = task["s3_key"]
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
att = await session.get(Attachment, att_id)
|
||||
if not att:
|
||||
return
|
||||
# ВАЖНО: создаем engine внутри функции с NullPool, чтобы избежать ошибки разных event loop
|
||||
engine = create_async_engine(DATABASE_URL, poolclass=NullPool)
|
||||
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
# идемпотентность: если уже обработано, пропускаем
|
||||
if att.status == "ready" and len(att.variants) == 3:
|
||||
return
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
att = await session.get(Attachment, att_id)
|
||||
if not att:
|
||||
return
|
||||
|
||||
try:
|
||||
# получаем оригинал
|
||||
obj = s3.get_object(Bucket=BUCKET, Key=s3_key)
|
||||
content = obj["Body"].read()
|
||||
img = Image.open(io.BytesIO(content))
|
||||
|
||||
# убираем прозрачность (композируем на белый) и разворачиваем по exif
|
||||
img = ImageOps.exif_transpose(img)
|
||||
if img.mode in ("RGBA", "LA"):
|
||||
background = Image.new("RGB", img.size, (255, 255, 255))
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
elif img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
# идемпотентность: если уже обработано, пропускаем
|
||||
if att.status == "ready" and len(att.variants or []) == 3:
|
||||
return
|
||||
|
||||
variants_meta = []
|
||||
try:
|
||||
# получаем оригинал
|
||||
obj = s3.get_object(Bucket=BUCKET, Key=s3_key)
|
||||
content = obj["Body"].read()
|
||||
img = Image.open(io.BytesIO(content))
|
||||
|
||||
# разворачиваем по exif и убираем прозрачность
|
||||
img = ImageOps.exif_transpose(img)
|
||||
if img.mode in ("RGBA", "LA"):
|
||||
background = Image.new("RGB", img.size, (255, 255, 255))
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
elif img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
|
||||
# --- 1. Thumbnail (256x256, вписанный) ---
|
||||
thumb = img.copy()
|
||||
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
||||
thumb_bytes = io.BytesIO()
|
||||
thumb.save(thumb_bytes, format="JPEG")
|
||||
thumb_data = thumb_bytes.getvalue()
|
||||
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_thumb.jpg", Body=thumb_data, ContentType="image/jpeg")
|
||||
variants_meta.append({
|
||||
"name": "thumbnail",
|
||||
"content_type": "image/jpeg",
|
||||
"width": thumb.width,
|
||||
"height": thumb.height,
|
||||
"size_bytes": len(thumb_data),
|
||||
"sha256": hashlib.sha256(thumb_data).hexdigest()
|
||||
})
|
||||
variants_meta = []
|
||||
|
||||
# --- 2. Crop (128x128, центрированный квадрат) ---
|
||||
crop_img = img.copy()
|
||||
width, height = crop_img.size
|
||||
min_dim = min(width, height)
|
||||
left = (width - min_dim) / 2
|
||||
top = (height - min_dim) / 2
|
||||
right = (width + min_dim) / 2
|
||||
bottom = (height + min_dim) / 2
|
||||
crop_img = crop_img.crop((left, top, right, bottom))
|
||||
crop_img = crop_img.resize((128, 128), Image.Resampling.LANCZOS)
|
||||
crop_bytes = io.BytesIO()
|
||||
crop_img.save(crop_bytes, format="JPEG")
|
||||
crop_data = crop_bytes.getvalue()
|
||||
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_crop.jpg", Body=crop_data, ContentType="image/jpeg")
|
||||
variants_meta.append({
|
||||
"name": "crop",
|
||||
"content_type": "image/jpeg",
|
||||
"width": 128,
|
||||
"height": 128,
|
||||
"size_bytes": len(crop_data),
|
||||
"sha256": hashlib.sha256(crop_data).hexdigest()
|
||||
})
|
||||
# 1. Thumbnail (256x256)
|
||||
thumb = img.copy()
|
||||
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
||||
thumb_bytes = io.BytesIO()
|
||||
thumb.save(thumb_bytes, format="JPEG")
|
||||
thumb_data = thumb_bytes.getvalue()
|
||||
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_thumb.jpg", Body=thumb_data, ContentType="image/jpeg")
|
||||
variants_meta.append({
|
||||
"name": "thumbnail", "content_type": "image/jpeg",
|
||||
"width": thumb.width, "height": thumb.height,
|
||||
"size_bytes": len(thumb_data), "sha256": hashlib.sha256(thumb_data).hexdigest()
|
||||
})
|
||||
|
||||
# --- 3. Optimized (WebP, до 1280x1280 без увеличения) ---
|
||||
opt_img = img.copy()
|
||||
opt_img.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
opt_bytes = io.BytesIO()
|
||||
opt_img.save(opt_bytes, format="WEBP", quality=85)
|
||||
opt_data = opt_bytes.getvalue()
|
||||
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_opt.webp", Body=opt_data, ContentType="image/webp")
|
||||
variants_meta.append({
|
||||
"name": "optimized",
|
||||
"content_type": "image/webp",
|
||||
"width": opt_img.width,
|
||||
"height": opt_img.height,
|
||||
"size_bytes": len(opt_data),
|
||||
"sha256": hashlib.sha256(opt_data).hexdigest()
|
||||
})
|
||||
# 2. Crop (128x128)
|
||||
crop_img = img.copy()
|
||||
width, height = crop_img.size
|
||||
min_dim = min(width, height)
|
||||
left = (width - min_dim) / 2
|
||||
top = (height - min_dim) / 2
|
||||
right = (width + min_dim) / 2
|
||||
bottom = (height + min_dim) / 2
|
||||
crop_img = crop_img.crop((left, top, right, bottom))
|
||||
crop_img = crop_img.resize((128, 128), Image.Resampling.LANCZOS)
|
||||
crop_bytes = io.BytesIO()
|
||||
crop_img.save(crop_bytes, format="JPEG")
|
||||
crop_data = crop_bytes.getvalue()
|
||||
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_crop.jpg", Body=crop_data, ContentType="image/jpeg")
|
||||
variants_meta.append({
|
||||
"name": "crop", "content_type": "image/jpeg",
|
||||
"width": 128, "height": 128,
|
||||
"size_bytes": len(crop_data), "sha256": hashlib.sha256(crop_data).hexdigest()
|
||||
})
|
||||
|
||||
# обновляем БД
|
||||
att.status = "ready"
|
||||
att.error_code = None
|
||||
att.variants = variants_meta
|
||||
att.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
# 3. Optimized (WebP)
|
||||
opt_img = img.copy()
|
||||
opt_img.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
opt_bytes = io.BytesIO()
|
||||
opt_img.save(opt_bytes, format="WEBP", quality=85)
|
||||
opt_data = opt_bytes.getvalue()
|
||||
s3.put_object(Bucket=BUCKET, Key=f"{s3_key}_opt.webp", Body=opt_data, ContentType="image/webp")
|
||||
variants_meta.append({
|
||||
"name": "optimized", "content_type": "image/webp",
|
||||
"width": opt_img.width, "height": opt_img.height,
|
||||
"size_bytes": len(opt_data), "sha256": hashlib.sha256(opt_data).hexdigest()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
att.status = "failed"
|
||||
att.error_code = "processing_failed"
|
||||
att.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
print(f"Error processing {att_id}: {e}")
|
||||
att.status = "ready"
|
||||
att.error_code = None
|
||||
att.variants = variants_meta
|
||||
att.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
att.status = "failed"
|
||||
att.error_code = "processing_failed"
|
||||
att.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
print(f"Error processing {att_id}: {e}", flush=True)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
def callback(ch, method, properties, body):
|
||||
task = json.loads(body)
|
||||
@@ -129,7 +124,7 @@ def main():
|
||||
channel.queue_declare(queue=QUEUE_NAME, durable=True)
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=callback)
|
||||
print("Worker started, waiting for messages...")
|
||||
print("Worker started, waiting for messages...", flush=True)
|
||||
channel.start_consuming()
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -88,6 +88,11 @@ services:
|
||||
condition: service_healthy
|
||||
migrator:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
|
||||
worker:
|
||||
build: .
|
||||
@@ -108,6 +113,11 @@ services:
|
||||
condition: service_healthy
|
||||
migrator:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1.0'
|
||||
memory: 1G
|
||||
|
||||
nginx:
|
||||
image: nginx:1.25-alpine
|
||||
|
||||
Reference in New Issue
Block a user