218 lines
9.3 KiB
Python
218 lines
9.3 KiB
Python
import os
|
|
import sys
|
|
import uuid
|
|
import hashlib
|
|
import asyncio
|
|
import logging
|
|
import tempfile
|
|
import json
|
|
from datetime import datetime, timezone, timedelta
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
from sqlalchemy.orm import sessionmaker
|
|
import aio_pika
|
|
from minio import Minio
|
|
from PIL import Image
|
|
import io
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db")
|
|
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
|
|
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
|
|
WORKER_ID = str(uuid.uuid4())[:8]
|
|
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
|
|
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
|
|
|
engine = create_async_engine(DATABASE_URL, echo=False)
|
|
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
|
|
|
|
async def process_image(body: bytes, att_id: str, chat_id: str):
|
|
img = Image.open(io.BytesIO(body)).convert("RGB")
|
|
width, height = img.size
|
|
variants_to_save = []
|
|
|
|
# 1. Thumbnail (вписан в 256x256)
|
|
thumb = img.copy()
|
|
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
|
thumb_bytes = io.BytesIO()
|
|
thumb.save(thumb_bytes, format="JPEG", quality=85)
|
|
thumb_data = thumb_bytes.getvalue()
|
|
thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
|
|
tmp.write(thumb_data)
|
|
s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg")
|
|
os.unlink(tmp.name)
|
|
variants_to_save.append({
|
|
"variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg",
|
|
"size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height,
|
|
"sha256": hashlib.sha256(thumb_data).hexdigest()
|
|
})
|
|
|
|
# 2. Crop (центральный квадрат 128x128)
|
|
min_dim = min(width, height)
|
|
left = (width - min_dim) // 2
|
|
top = (height - min_dim) // 2
|
|
crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS)
|
|
crop_bytes = io.BytesIO()
|
|
crop.save(crop_bytes, format="JPEG", quality=85)
|
|
crop_data = crop_bytes.getvalue()
|
|
crop_key = f"variants/{chat_id}/{att_id}/crop.jpg"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
|
|
tmp.write(crop_data)
|
|
s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg")
|
|
os.unlink(tmp.name)
|
|
variants_to_save.append({
|
|
"variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg",
|
|
"size_bytes": len(crop_data), "width": 128, "height": 128,
|
|
"sha256": hashlib.sha256(crop_data).hexdigest()
|
|
})
|
|
|
|
# 3. Optimized WebP (вписан в 1280x1280)
|
|
opt = img.copy()
|
|
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
|
opt_bytes = io.BytesIO()
|
|
opt.save(opt_bytes, format="WEBP", quality=80)
|
|
opt_data = opt_bytes.getvalue()
|
|
opt_key = f"variants/{chat_id}/{att_id}/optimized.webp"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp:
|
|
tmp.write(opt_data)
|
|
s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp")
|
|
os.unlink(tmp.name)
|
|
variants_to_save.append({
|
|
"variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp",
|
|
"size_bytes": len(opt_data), "width": opt.width, "height": opt.height,
|
|
"sha256": hashlib.sha256(opt_data).hexdigest()
|
|
})
|
|
|
|
# 4. Сохраняем оригинал в конечную папку
|
|
ext = "png" if b"png" in body[:8] else "jpeg"
|
|
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
|
|
content_type = "image/png" if ext == "png" else "image/jpeg"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp:
|
|
tmp.write(body)
|
|
s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type)
|
|
os.unlink(tmp.name)
|
|
|
|
return final_key, variants_to_save
|
|
|
|
async def process_message(message: aio_pika.IncomingMessage):
|
|
async with message.process(requeue=False):
|
|
try:
|
|
payload = message.body.decode()
|
|
data = json.loads(payload)
|
|
job_id = data["job_id"]
|
|
att_id = data["attachment_id"]
|
|
attempt = data["attempt"]
|
|
|
|
logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}")
|
|
|
|
async with AsyncSessionLocal() as db:
|
|
from app.models import AttachmentJob, Attachment, AttachmentVariant
|
|
|
|
# 1. Блокируем задачу для обработки (Claim)
|
|
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
|
result = await db.execute(stmt)
|
|
job = result.scalar_one_or_none()
|
|
|
|
if not job:
|
|
logger.warning(f"Job {job_id} not found")
|
|
return
|
|
|
|
if job.status in ("ready", "failed"):
|
|
logger.info(f"Job {job_id} already {job.status}, skipping")
|
|
return
|
|
|
|
if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc):
|
|
logger.info(f"Job {job_id} is already being processed by another worker")
|
|
return
|
|
|
|
now = datetime.now(timezone.utc)
|
|
job.status = "processing"
|
|
job.attempt = attempt
|
|
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
|
|
job.lease_owner = WORKER_ID
|
|
await db.commit()
|
|
|
|
# 2. Получаем метаданные вложения
|
|
att_stmt = select(Attachment).where(Attachment.id == att_id)
|
|
att_result = await db.execute(att_stmt)
|
|
attachment = att_result.scalar_one_or_none()
|
|
|
|
if not attachment or not attachment.staging_key:
|
|
job.status = "failed"
|
|
job.error_code = "missing_staging"
|
|
await db.commit()
|
|
return
|
|
|
|
staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления
|
|
|
|
# 3. Скачиваем из staging
|
|
response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete)
|
|
body = response.read()
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
# 4. Обрабатываем изображение
|
|
final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id))
|
|
|
|
# 5. Фиксируем результат в БД
|
|
attachment.status = "ready"
|
|
attachment.s3_key = final_key
|
|
attachment.staging_key = None # Очищаем staging
|
|
|
|
job.status = "ready"
|
|
|
|
for v_data in variants_data:
|
|
variant = AttachmentVariant(
|
|
attachment_id=attachment.id,
|
|
**v_data
|
|
)
|
|
db.add(variant)
|
|
|
|
await db.commit()
|
|
logger.info(f"Job {job_id} completed successfully")
|
|
|
|
# 6. Удаляем staging из S3
|
|
s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing message: {e}")
|
|
try:
|
|
async with AsyncSessionLocal() as db:
|
|
from app.models import AttachmentJob
|
|
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
|
result = await db.execute(stmt)
|
|
job = result.scalar_one_or_none()
|
|
if job:
|
|
if job.attempt >= JOB_MAX_ATTEMPTS:
|
|
job.status = "failed"
|
|
job.error_code = "retry_exhausted"
|
|
else:
|
|
job.status = "queued"
|
|
job.attempt += 1
|
|
job.lease_until = None
|
|
await db.commit()
|
|
except Exception as db_err:
|
|
logger.error(f"Failed to update job status after error: {db_err}")
|
|
|
|
async def main():
|
|
logger.info("Starting worker...")
|
|
connection = await aio_pika.connect_robust(BROKER_URL)
|
|
channel = await connection.channel()
|
|
|
|
queue = await channel.declare_queue("attachment.process.v1", durable=True)
|
|
|
|
logger.info("Waiting for messages...")
|
|
await queue.consume(process_message)
|
|
|
|
await asyncio.Future()
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |