Files
mtusi2026lab5/app/worker.py
T

218 lines
9.3 KiB
Python

import os
import sys
import uuid
import hashlib
import asyncio
import logging
import tempfile
import json
from datetime import datetime, timezone, timedelta
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
import aio_pika
from minio import Minio
from PIL import Image
import io
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db")
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
WORKER_ID = str(uuid.uuid4())[:8]
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
engine = create_async_engine(DATABASE_URL, echo=False)
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
async def process_image(body: bytes, att_id: str, chat_id: str):
img = Image.open(io.BytesIO(body)).convert("RGB")
width, height = img.size
variants_to_save = []
# 1. Thumbnail (вписан в 256x256)
thumb = img.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumb_bytes = io.BytesIO()
thumb.save(thumb_bytes, format="JPEG", quality=85)
thumb_data = thumb_bytes.getvalue()
thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
tmp.write(thumb_data)
s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg",
"size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height,
"sha256": hashlib.sha256(thumb_data).hexdigest()
})
# 2. Crop (центральный квадрат 128x128)
min_dim = min(width, height)
left = (width - min_dim) // 2
top = (height - min_dim) // 2
crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS)
crop_bytes = io.BytesIO()
crop.save(crop_bytes, format="JPEG", quality=85)
crop_data = crop_bytes.getvalue()
crop_key = f"variants/{chat_id}/{att_id}/crop.jpg"
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
tmp.write(crop_data)
s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg",
"size_bytes": len(crop_data), "width": 128, "height": 128,
"sha256": hashlib.sha256(crop_data).hexdigest()
})
# 3. Optimized WebP (вписан в 1280x1280)
opt = img.copy()
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
opt_bytes = io.BytesIO()
opt.save(opt_bytes, format="WEBP", quality=80)
opt_data = opt_bytes.getvalue()
opt_key = f"variants/{chat_id}/{att_id}/optimized.webp"
with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp:
tmp.write(opt_data)
s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp",
"size_bytes": len(opt_data), "width": opt.width, "height": opt.height,
"sha256": hashlib.sha256(opt_data).hexdigest()
})
# 4. Сохраняем оригинал в конечную папку
ext = "png" if b"png" in body[:8] else "jpeg"
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
content_type = "image/png" if ext == "png" else "image/jpeg"
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp:
tmp.write(body)
s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type)
os.unlink(tmp.name)
return final_key, variants_to_save
async def process_message(message: aio_pika.IncomingMessage):
async with message.process(requeue=False):
try:
payload = message.body.decode()
data = json.loads(payload)
job_id = data["job_id"]
att_id = data["attachment_id"]
attempt = data["attempt"]
logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}")
async with AsyncSessionLocal() as db:
from app.models import AttachmentJob, Attachment, AttachmentVariant
# 1. Блокируем задачу для обработки (Claim)
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
result = await db.execute(stmt)
job = result.scalar_one_or_none()
if not job:
logger.warning(f"Job {job_id} not found")
return
if job.status in ("ready", "failed"):
logger.info(f"Job {job_id} already {job.status}, skipping")
return
if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc):
logger.info(f"Job {job_id} is already being processed by another worker")
return
now = datetime.now(timezone.utc)
job.status = "processing"
job.attempt = attempt
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
job.lease_owner = WORKER_ID
await db.commit()
# 2. Получаем метаданные вложения
att_stmt = select(Attachment).where(Attachment.id == att_id)
att_result = await db.execute(att_stmt)
attachment = att_result.scalar_one_or_none()
if not attachment or not attachment.staging_key:
job.status = "failed"
job.error_code = "missing_staging"
await db.commit()
return
staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления
# 3. Скачиваем из staging
response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete)
body = response.read()
response.close()
response.release_conn()
# 4. Обрабатываем изображение
final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id))
# 5. Фиксируем результат в БД
attachment.status = "ready"
attachment.s3_key = final_key
attachment.staging_key = None # Очищаем staging
job.status = "ready"
for v_data in variants_data:
variant = AttachmentVariant(
attachment_id=attachment.id,
**v_data
)
db.add(variant)
await db.commit()
logger.info(f"Job {job_id} completed successfully")
# 6. Удаляем staging из S3
s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete)
except Exception as e:
logger.error(f"Error processing message: {e}")
try:
async with AsyncSessionLocal() as db:
from app.models import AttachmentJob
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
result = await db.execute(stmt)
job = result.scalar_one_or_none()
if job:
if job.attempt >= JOB_MAX_ATTEMPTS:
job.status = "failed"
job.error_code = "retry_exhausted"
else:
job.status = "queued"
job.attempt += 1
job.lease_until = None
await db.commit()
except Exception as db_err:
logger.error(f"Failed to update job status after error: {db_err}")
async def main():
logger.info("Starting worker...")
connection = await aio_pika.connect_robust(BROKER_URL)
channel = await connection.channel()
queue = await channel.declare_queue("attachment.process.v1", durable=True)
logger.info("Waiting for messages...")
await queue.consume(process_message)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())