Files
mtusi2026lab5/app/worker.py
T

347 lines
14 KiB
Python

import os
import io
import json
import uuid
import hashlib
import asyncio
import logging
import tempfile
from datetime import datetime, timezone, timedelta
import aio_pika
from minio import Minio
from PIL import Image
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
from sqlalchemy.dialects.postgresql import insert as pg_insert
from app.models import Attachment, AttachmentJob, AttachmentVariant
from app.broker import (
get_channel, declare_topology, publish_retry, choose_retry_tier, MAIN_QUEUE,
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger(__name__)
DATABASE_URL = os.getenv("DATABASE_URL")
BROKER_URL = os.getenv("BROKER_URL")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
WORKER_ID = str(uuid.uuid4())[:8]
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2"))
MAX_PIXELS = 40_000_000
MAX_BYTES = 10 * 1024 * 1024
engine = create_async_engine(DATABASE_URL, echo=False)
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
class PermanentError(Exception):
"""Повтор бессмысленен: битый файл, слишком большое изображение, нет staging."""
def __init__(self, code, message=""):
self.code = code
super().__init__(f"{code}: {message}")
class TemporaryError(Exception):
"""Повтор имеет смысл: S3 недоступен, таймаут, сеть."""
def __init__(self, code, message=""):
self.code = code
super().__init__(f"{code}: {message}")
def s3_put(key: str, data: bytes, content_type: str):
with tempfile.NamedTemporaryFile(delete=False) as tmp:
tmp.write(data)
path = tmp.name
try:
s3_client.fput_object(S3_BUCKET_NAME, key, path, content_type=content_type)
except Exception as e:
raise TemporaryError("storage_unavailable", str(e))
finally:
os.unlink(path)
def build_variants(body: bytes, att_id: str, chat_id: str):
"""CPU-bound часть. Только worker, никогда API."""
if len(body) > MAX_BYTES:
raise PermanentError("payload_too_large")
try:
img = Image.open(io.BytesIO(body))
img.load()
img = img.convert("RGB") # EXIF/ICC не переносим — энкодер получает только пиксели
except Exception as e:
raise PermanentError("invalid_image", str(e))
w, h = img.size
if w * h > MAX_PIXELS:
raise PermanentError("image_too_large")
out = []
thumb = img.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85)
data = buf.getvalue()
key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
s3_put(key, data, "image/jpeg")
out.append(dict(variant_name="thumbnail", s3_key=key, content_type="image/jpeg",
size_bytes=len(data), width=thumb.width, height=thumb.height,
sha256=hashlib.sha256(data).hexdigest()))
side = min(w, h)
left, top = (w - side) // 2, (h - side) // 2
crop = img.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85)
data = buf.getvalue()
key = f"variants/{chat_id}/{att_id}/crop.jpg"
s3_put(key, data, "image/jpeg")
out.append(dict(variant_name="crop", s3_key=key, content_type="image/jpeg",
size_bytes=len(data), width=128, height=128,
sha256=hashlib.sha256(data).hexdigest()))
opt = img.copy()
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=80, method=4)
data = buf.getvalue()
key = f"variants/{chat_id}/{att_id}/optimized.webp"
s3_put(key, data, "image/webp")
out.append(dict(variant_name="optimized", s3_key=key, content_type="image/webp",
size_bytes=len(data), width=opt.width, height=opt.height,
sha256=hashlib.sha256(data).hexdigest()))
ext = "png" if body[:8].startswith(b"\x89PNG") else "jpeg"
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
s3_put(final_key, body, "image/png" if ext == "png" else "image/jpeg")
return final_key, out
async def handle_job(job_id: str, att_id: str, attempt: int):
"""Возвращает (outcome, error_code). outcome: done|noop|permanent|temporary"""
async with AsyncSessionLocal() as db:
job = (await db.execute(
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
)).scalar_one_or_none()
if job is None:
return "noop", None
if job.status in ("ready", "failed"):
logger.info("Job %s already %s -> no-op ack", job_id, job.status)
return "noop", None
if attempt < job.attempt:
logger.info("Job %s stale delivery (msg attempt=%s < db attempt=%s) -> no-op", job_id, attempt, job.attempt)
return "noop", None
now = datetime.now(timezone.utc)
if job.status == "processing" and job.lease_until and job.lease_until > now and not job.lease_owner == WORKER_ID:
logger.info("Job %s leased by %s until %s -> no-op", job_id, job.lease_owner, job.lease_until)
return "noop", None
job.status = "processing"
job.attempt = attempt
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
job.lease_owner = WORKER_ID
att = (await db.execute(select(Attachment).where(Attachment.id == att_id))).scalar_one_or_none()
if att is None:
return "noop", None
att.status = "processing"
await db.commit()
staging_key = att.staging_key
if not staging_key:
return "permanent", "missing_staging"
try:
resp = s3_client.get_object(S3_BUCKET_NAME, staging_key)
try:
body = resp.read()
finally:
resp.close(); resp.release_conn()
except Exception as e:
return "temporary", "storage_unavailable"
if hashlib.sha256(body).hexdigest() != att.sha256:
return "permanent", "checksum_mismatch"
try:
final_key, variants = await asyncio.wait_for(
asyncio.to_thread(build_variants, body, str(att.id), str(att.chat_id)),
timeout=int(os.getenv("JOB_TIMEOUT_SECONDS", "60")),
)
except asyncio.TimeoutError:
return "temporary", "processing_timeout"
except PermanentError as e:
return "permanent", e.code
except TemporaryError as e:
return "temporary", e.code
except Exception as e:
logger.exception("Unexpected processing failure")
return "temporary", "processing_error"
# Идемпотентная фиксация: устаревшая попытка не перезапишет результат
res = await db.execute(
update(AttachmentJob)
.where(AttachmentJob.job_id == job_id,
AttachmentJob.attempt == attempt,
AttachmentJob.status == "processing")
.values(status="ready", lease_until=None)
)
if res.rowcount == 0:
logger.info("Job %s attempt %s lost ownership -> discard result", job_id, attempt)
return "noop", None
for v in variants:
await db.execute(
pg_insert(AttachmentVariant)
.values(attachment_id=att.id, **v)
.on_conflict_do_nothing(index_elements=["attachment_id", "variant_name"])
)
await db.execute(
update(Attachment).where(Attachment.id == att.id)
.values(status="ready", s3_key=final_key, staging_key=None, error_code=None)
)
await db.commit()
# Очистка staging — только после безопасной фиксации
try:
s3_client.remove_object(S3_BUCKET_NAME, staging_key)
except Exception as e:
logger.warning("Staging cleanup failed for %s: %s", staging_key, e)
logger.info("Job %s completed successfully (attempt %s)", job_id, attempt)
return "done", None
async def mark_failed(job_id: str, code: str):
async with AsyncSessionLocal() as db:
job = (await db.execute(select(AttachmentJob).where(AttachmentJob.job_id == job_id))).scalar_one_or_none()
if job is None or job.status == "ready":
return
await db.execute(update(AttachmentJob).where(AttachmentJob.job_id == job_id)
.values(status="failed", lease_until=None))
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
.values(status="failed", error_code=code))
await db.commit()
logger.warning("Job %s -> failed (%s)", job_id, code)
async def schedule_retry(job_id: str, payload: dict) -> int:
"""Возвращает выбранный tier; -1 если бюджет попыток исчерпан."""
async with AsyncSessionLocal() as db:
job = (await db.execute(
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
)).scalar_one_or_none()
if job is None or job.status == "ready":
return -1
if job.attempt >= JOB_MAX_ATTEMPTS:
return -1
next_attempt = job.attempt + 1
tier = choose_retry_tier(job.attempt)
job.status = "queued"
job.attempt = next_attempt
job.lease_until = None
job.lease_owner = None
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id).values(status="queued"))
await db.commit()
await publish_retry({**payload, "attempt": next_attempt}, tier)
return tier
async def process_message(message: aio_pika.IncomingMessage):
# Ручные подтверждения: без message.process(), чтобы не было двойного ack
try:
data = json.loads(message.body)
job_id, att_id, attempt = data["job_id"], data["attachment_id"], int(data["attempt"])
uuid.UUID(job_id); uuid.UUID(att_id)
except Exception:
logger.error("Malformed message -> DLQ")
await message.nack(requeue=False)
return
try:
outcome, code = await handle_job(job_id, att_id, attempt)
except Exception:
logger.exception("Handler crashed for job %s", job_id)
outcome, code = "temporary", "internal_error"
try:
if outcome in ("done", "noop"):
await message.ack()
return
if outcome == "permanent":
await mark_failed(job_id, code)
await message.nack(requeue=False) # -> DLX -> DLQ
return
# temporary
tier = await schedule_retry(job_id, data)
if tier < 0:
await mark_failed(job_id, "retry_exhausted")
await message.nack(requeue=False) # -> DLQ
else:
logger.info("Job %s temporary error (%s), retry in ~%ss", job_id, code, tier)
await message.ack() # копия уже в отложенной очереди
except Exception:
logger.exception("Failed to finalize delivery for job %s -> requeue", job_id)
await message.nack(requeue=True)
async def sweeper():
"""Зависшие processing (lease истёк) возвращаем в работу."""
while True:
try:
await asyncio.sleep(5)
async with AsyncSessionLocal() as db:
now = datetime.now(timezone.utc)
rows = (await db.execute(
select(AttachmentJob).where(
AttachmentJob.status == "processing",
AttachmentJob.lease_until.isnot(None),
AttachmentJob.lease_until < now,
).limit(20)
)).scalars().all()
for job in rows:
if job.attempt >= JOB_MAX_ATTEMPTS:
continue
payload = {"job_id": str(job.job_id), "attachment_id": str(job.attachment_id),
"attempt": job.attempt + 1}
job.status = "queued"
job.attempt += 1
job.lease_until = None
job.lease_owner = None
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
.values(status="queued"))
await db.commit()
await publish_retry(payload, choose_retry_tier(1))
logger.warning("Sweeper: lease expired for job %s -> retry attempt %s", job.job_id, job.attempt)
except asyncio.CancelledError:
raise
except Exception as e:
logger.error("Sweeper error: %s", e)
async def main():
logger.info("Starting worker %s (concurrency/prefetch=%s)", WORKER_ID, WORKER_CONCURRENCY)
channel = await get_channel()
await declare_topology(channel)
await channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
queue = await channel.get_queue(MAIN_QUEUE)
await queue.consume(process_message)
asyncio.create_task(sweeper())
logger.info("Waiting for messages on %s ...", MAIN_QUEUE)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())