347 lines
14 KiB
Python
347 lines
14 KiB
Python
import os
|
|
import io
|
|
import json
|
|
import uuid
|
|
import hashlib
|
|
import asyncio
|
|
import logging
|
|
import tempfile
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
import aio_pika
|
|
from minio import Minio
|
|
from PIL import Image
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
from sqlalchemy.orm import sessionmaker
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
from app.models import Attachment, AttachmentJob, AttachmentVariant
|
|
from app.broker import (
|
|
get_channel, declare_topology, publish_retry, choose_retry_tier, MAIN_QUEUE,
|
|
)
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DATABASE_URL = os.getenv("DATABASE_URL")
|
|
BROKER_URL = os.getenv("BROKER_URL")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
|
|
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
|
|
|
|
WORKER_ID = str(uuid.uuid4())[:8]
|
|
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
|
|
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
|
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2"))
|
|
MAX_PIXELS = 40_000_000
|
|
MAX_BYTES = 10 * 1024 * 1024
|
|
|
|
engine = create_async_engine(DATABASE_URL, echo=False)
|
|
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
|
|
|
|
|
|
class PermanentError(Exception):
|
|
"""Повтор бессмысленен: битый файл, слишком большое изображение, нет staging."""
|
|
def __init__(self, code, message=""):
|
|
self.code = code
|
|
super().__init__(f"{code}: {message}")
|
|
|
|
|
|
class TemporaryError(Exception):
|
|
"""Повтор имеет смысл: S3 недоступен, таймаут, сеть."""
|
|
def __init__(self, code, message=""):
|
|
self.code = code
|
|
super().__init__(f"{code}: {message}")
|
|
|
|
|
|
def s3_put(key: str, data: bytes, content_type: str):
|
|
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
|
tmp.write(data)
|
|
path = tmp.name
|
|
try:
|
|
s3_client.fput_object(S3_BUCKET_NAME, key, path, content_type=content_type)
|
|
except Exception as e:
|
|
raise TemporaryError("storage_unavailable", str(e))
|
|
finally:
|
|
os.unlink(path)
|
|
|
|
|
|
def build_variants(body: bytes, att_id: str, chat_id: str):
|
|
"""CPU-bound часть. Только worker, никогда API."""
|
|
if len(body) > MAX_BYTES:
|
|
raise PermanentError("payload_too_large")
|
|
try:
|
|
img = Image.open(io.BytesIO(body))
|
|
img.load()
|
|
img = img.convert("RGB") # EXIF/ICC не переносим — энкодер получает только пиксели
|
|
except Exception as e:
|
|
raise PermanentError("invalid_image", str(e))
|
|
|
|
w, h = img.size
|
|
if w * h > MAX_PIXELS:
|
|
raise PermanentError("image_too_large")
|
|
|
|
out = []
|
|
|
|
thumb = img.copy()
|
|
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
|
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85)
|
|
data = buf.getvalue()
|
|
key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
|
|
s3_put(key, data, "image/jpeg")
|
|
out.append(dict(variant_name="thumbnail", s3_key=key, content_type="image/jpeg",
|
|
size_bytes=len(data), width=thumb.width, height=thumb.height,
|
|
sha256=hashlib.sha256(data).hexdigest()))
|
|
|
|
side = min(w, h)
|
|
left, top = (w - side) // 2, (h - side) // 2
|
|
crop = img.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
|
|
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85)
|
|
data = buf.getvalue()
|
|
key = f"variants/{chat_id}/{att_id}/crop.jpg"
|
|
s3_put(key, data, "image/jpeg")
|
|
out.append(dict(variant_name="crop", s3_key=key, content_type="image/jpeg",
|
|
size_bytes=len(data), width=128, height=128,
|
|
sha256=hashlib.sha256(data).hexdigest()))
|
|
|
|
opt = img.copy()
|
|
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
|
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=80, method=4)
|
|
data = buf.getvalue()
|
|
key = f"variants/{chat_id}/{att_id}/optimized.webp"
|
|
s3_put(key, data, "image/webp")
|
|
out.append(dict(variant_name="optimized", s3_key=key, content_type="image/webp",
|
|
size_bytes=len(data), width=opt.width, height=opt.height,
|
|
sha256=hashlib.sha256(data).hexdigest()))
|
|
|
|
ext = "png" if body[:8].startswith(b"\x89PNG") else "jpeg"
|
|
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
|
|
s3_put(final_key, body, "image/png" if ext == "png" else "image/jpeg")
|
|
return final_key, out
|
|
|
|
|
|
async def handle_job(job_id: str, att_id: str, attempt: int):
|
|
"""Возвращает (outcome, error_code). outcome: done|noop|permanent|temporary"""
|
|
async with AsyncSessionLocal() as db:
|
|
job = (await db.execute(
|
|
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
|
)).scalar_one_or_none()
|
|
|
|
if job is None:
|
|
return "noop", None
|
|
if job.status in ("ready", "failed"):
|
|
logger.info("Job %s already %s -> no-op ack", job_id, job.status)
|
|
return "noop", None
|
|
if attempt < job.attempt:
|
|
logger.info("Job %s stale delivery (msg attempt=%s < db attempt=%s) -> no-op", job_id, attempt, job.attempt)
|
|
return "noop", None
|
|
|
|
now = datetime.now(timezone.utc)
|
|
if job.status == "processing" and job.lease_until and job.lease_until > now and not job.lease_owner == WORKER_ID:
|
|
logger.info("Job %s leased by %s until %s -> no-op", job_id, job.lease_owner, job.lease_until)
|
|
return "noop", None
|
|
|
|
job.status = "processing"
|
|
job.attempt = attempt
|
|
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
|
|
job.lease_owner = WORKER_ID
|
|
|
|
att = (await db.execute(select(Attachment).where(Attachment.id == att_id))).scalar_one_or_none()
|
|
if att is None:
|
|
return "noop", None
|
|
att.status = "processing"
|
|
await db.commit()
|
|
|
|
staging_key = att.staging_key
|
|
if not staging_key:
|
|
return "permanent", "missing_staging"
|
|
|
|
try:
|
|
resp = s3_client.get_object(S3_BUCKET_NAME, staging_key)
|
|
try:
|
|
body = resp.read()
|
|
finally:
|
|
resp.close(); resp.release_conn()
|
|
except Exception as e:
|
|
return "temporary", "storage_unavailable"
|
|
|
|
if hashlib.sha256(body).hexdigest() != att.sha256:
|
|
return "permanent", "checksum_mismatch"
|
|
|
|
try:
|
|
final_key, variants = await asyncio.wait_for(
|
|
asyncio.to_thread(build_variants, body, str(att.id), str(att.chat_id)),
|
|
timeout=int(os.getenv("JOB_TIMEOUT_SECONDS", "60")),
|
|
)
|
|
except asyncio.TimeoutError:
|
|
return "temporary", "processing_timeout"
|
|
except PermanentError as e:
|
|
return "permanent", e.code
|
|
except TemporaryError as e:
|
|
return "temporary", e.code
|
|
except Exception as e:
|
|
logger.exception("Unexpected processing failure")
|
|
return "temporary", "processing_error"
|
|
|
|
# Идемпотентная фиксация: устаревшая попытка не перезапишет результат
|
|
res = await db.execute(
|
|
update(AttachmentJob)
|
|
.where(AttachmentJob.job_id == job_id,
|
|
AttachmentJob.attempt == attempt,
|
|
AttachmentJob.status == "processing")
|
|
.values(status="ready", lease_until=None)
|
|
)
|
|
if res.rowcount == 0:
|
|
logger.info("Job %s attempt %s lost ownership -> discard result", job_id, attempt)
|
|
return "noop", None
|
|
|
|
for v in variants:
|
|
await db.execute(
|
|
pg_insert(AttachmentVariant)
|
|
.values(attachment_id=att.id, **v)
|
|
.on_conflict_do_nothing(index_elements=["attachment_id", "variant_name"])
|
|
)
|
|
|
|
await db.execute(
|
|
update(Attachment).where(Attachment.id == att.id)
|
|
.values(status="ready", s3_key=final_key, staging_key=None, error_code=None)
|
|
)
|
|
await db.commit()
|
|
|
|
# Очистка staging — только после безопасной фиксации
|
|
try:
|
|
s3_client.remove_object(S3_BUCKET_NAME, staging_key)
|
|
except Exception as e:
|
|
logger.warning("Staging cleanup failed for %s: %s", staging_key, e)
|
|
|
|
logger.info("Job %s completed successfully (attempt %s)", job_id, attempt)
|
|
return "done", None
|
|
|
|
|
|
async def mark_failed(job_id: str, code: str):
|
|
async with AsyncSessionLocal() as db:
|
|
job = (await db.execute(select(AttachmentJob).where(AttachmentJob.job_id == job_id))).scalar_one_or_none()
|
|
if job is None or job.status == "ready":
|
|
return
|
|
await db.execute(update(AttachmentJob).where(AttachmentJob.job_id == job_id)
|
|
.values(status="failed", lease_until=None))
|
|
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
|
|
.values(status="failed", error_code=code))
|
|
await db.commit()
|
|
logger.warning("Job %s -> failed (%s)", job_id, code)
|
|
|
|
|
|
async def schedule_retry(job_id: str, payload: dict) -> int:
|
|
"""Возвращает выбранный tier; -1 если бюджет попыток исчерпан."""
|
|
async with AsyncSessionLocal() as db:
|
|
job = (await db.execute(
|
|
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
|
)).scalar_one_or_none()
|
|
if job is None or job.status == "ready":
|
|
return -1
|
|
if job.attempt >= JOB_MAX_ATTEMPTS:
|
|
return -1
|
|
next_attempt = job.attempt + 1
|
|
tier = choose_retry_tier(job.attempt)
|
|
job.status = "queued"
|
|
job.attempt = next_attempt
|
|
job.lease_until = None
|
|
job.lease_owner = None
|
|
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id).values(status="queued"))
|
|
await db.commit()
|
|
|
|
await publish_retry({**payload, "attempt": next_attempt}, tier)
|
|
return tier
|
|
|
|
|
|
async def process_message(message: aio_pika.IncomingMessage):
|
|
# Ручные подтверждения: без message.process(), чтобы не было двойного ack
|
|
try:
|
|
data = json.loads(message.body)
|
|
job_id, att_id, attempt = data["job_id"], data["attachment_id"], int(data["attempt"])
|
|
uuid.UUID(job_id); uuid.UUID(att_id)
|
|
except Exception:
|
|
logger.error("Malformed message -> DLQ")
|
|
await message.nack(requeue=False)
|
|
return
|
|
|
|
try:
|
|
outcome, code = await handle_job(job_id, att_id, attempt)
|
|
except Exception:
|
|
logger.exception("Handler crashed for job %s", job_id)
|
|
outcome, code = "temporary", "internal_error"
|
|
|
|
try:
|
|
if outcome in ("done", "noop"):
|
|
await message.ack()
|
|
return
|
|
|
|
if outcome == "permanent":
|
|
await mark_failed(job_id, code)
|
|
await message.nack(requeue=False) # -> DLX -> DLQ
|
|
return
|
|
|
|
# temporary
|
|
tier = await schedule_retry(job_id, data)
|
|
if tier < 0:
|
|
await mark_failed(job_id, "retry_exhausted")
|
|
await message.nack(requeue=False) # -> DLQ
|
|
else:
|
|
logger.info("Job %s temporary error (%s), retry in ~%ss", job_id, code, tier)
|
|
await message.ack() # копия уже в отложенной очереди
|
|
except Exception:
|
|
logger.exception("Failed to finalize delivery for job %s -> requeue", job_id)
|
|
await message.nack(requeue=True)
|
|
|
|
|
|
async def sweeper():
|
|
"""Зависшие processing (lease истёк) возвращаем в работу."""
|
|
while True:
|
|
try:
|
|
await asyncio.sleep(5)
|
|
async with AsyncSessionLocal() as db:
|
|
now = datetime.now(timezone.utc)
|
|
rows = (await db.execute(
|
|
select(AttachmentJob).where(
|
|
AttachmentJob.status == "processing",
|
|
AttachmentJob.lease_until.isnot(None),
|
|
AttachmentJob.lease_until < now,
|
|
).limit(20)
|
|
)).scalars().all()
|
|
for job in rows:
|
|
if job.attempt >= JOB_MAX_ATTEMPTS:
|
|
continue
|
|
payload = {"job_id": str(job.job_id), "attachment_id": str(job.attachment_id),
|
|
"attempt": job.attempt + 1}
|
|
job.status = "queued"
|
|
job.attempt += 1
|
|
job.lease_until = None
|
|
job.lease_owner = None
|
|
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
|
|
.values(status="queued"))
|
|
await db.commit()
|
|
await publish_retry(payload, choose_retry_tier(1))
|
|
logger.warning("Sweeper: lease expired for job %s -> retry attempt %s", job.job_id, job.attempt)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
logger.error("Sweeper error: %s", e)
|
|
|
|
|
|
async def main():
|
|
logger.info("Starting worker %s (concurrency/prefetch=%s)", WORKER_ID, WORKER_CONCURRENCY)
|
|
channel = await get_channel()
|
|
await declare_topology(channel)
|
|
await channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
|
|
queue = await channel.get_queue(MAIN_QUEUE)
|
|
await queue.consume(process_message)
|
|
asyncio.create_task(sweeper())
|
|
logger.info("Waiting for messages on %s ...", MAIN_QUEUE)
|
|
await asyncio.Future()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |