Files
mtusi2026lab5/src/worker.py
T

265 lines
8.7 KiB
Python

from __future__ import annotations
import io
import logging
import os
import time
import random
from PIL import Image, ImageOps
from . import broker, storage
from .db import init_pool, close_pool, transaction
from .store import store, iso
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("messenger.worker")
THUMB_SIZE = (256, 256)
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
def _load_original(staging_key: str) -> bytes:
return storage.get_object(staging_key)
def _make_thumbnail(data: bytes) -> tuple[bytes, int, int]:
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
background = Image.new("RGB", img.size, (255, 255, 255))
rgba = img.convert("RGBA")
background.paste(rgba, mask=rgba.split()[-1])
img = background
else:
img = img.convert("RGB")
img.thumbnail(THUMB_SIZE, Image.LANCZOS)
w, h = img.size
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85, optimize=True)
return buf.getvalue(), w, h
def _make_crop(data: bytes, size: int = 128) -> tuple[bytes, int, int]:
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
background = Image.new("RGB", img.size, (255, 255, 255))
rgba = img.convert("RGBA")
background.paste(rgba, mask=rgba.split()[-1])
img = background
else:
img = img.convert("RGB")
w, h = img.size
side = min(w, h)
left = (w - side) // 2
top = (h - side) // 2
img = img.crop((left, top, left + side, top + side))
img = img.resize((size, size), Image.LANCZOS)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85, optimize=True)
return buf.getvalue(), size, size
def _make_optimized(data: bytes, max_side: int = 1280) -> tuple[bytes, int, int]:
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
background = Image.new("RGB", img.size, (255, 255, 255))
rgba = img.convert("RGBA")
background.paste(rgba, mask=rgba.split()[-1])
img = background
else:
img = img.convert("RGB")
img.thumbnail((max_side, max_side), Image.LANCZOS)
w, h = img.size
buf = io.BytesIO()
img.save(buf, format="WEBP", quality=80, method=4)
return buf.getvalue(), w, h
def _handle(job: dict) -> None:
attachment_id = job["attachment_id"]
attempt = int(job.get("attempt", 1))
job_id = job["job_id"]
log.info("processing attachment %s attempt %d (job %s)", attachment_id, attempt, job_id)
with transaction() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE jobs
SET status = 'processing',
lease_until = NOW() + INTERVAL '60 seconds'
WHERE job_id = %s AND status IN ('published', 'pending')
RETURNING job_id
""",
(job_id,),
)
acquired = cur.fetchone()
if acquired is None:
log.info("job %s already acquired or done, skip", job_id)
return
with transaction() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE attachments SET status='processing', updated_at=NOW() "
"WHERE id=%s AND status='queued'",
(attachment_id,),
)
cur.execute(
"SELECT id, chat_id, owner_id, filename, content_type, "
"size_bytes, sha256, s3_key, staging_key, status "
"FROM attachments WHERE id = %s",
(attachment_id,),
)
row = cur.fetchone()
if row is None:
log.error("attachment %s not found, skip", attachment_id)
return
if row["status"] == "ready":
log.info("attachment %s already ready, no-op", attachment_id)
return
staging_key = row.get("staging_key") or row["s3_key"]
data = _load_original(staging_key)
thumb_bytes, tw, th = _make_thumbnail(data)
crop_bytes, cw, ch = _make_crop(data, 128)
opt_bytes, ow, oh = _make_optimized(data, 1280)
ext = row["content_type"].split("/")[-1]
final_key = f"originals/{attachment_id}.{ext}"
if staging_key != final_key:
storage.put_object(final_key, data, row["content_type"])
thumb_key = f"thumbnails/{attachment_id}.jpg"
thumb_sha = storage.put_object(thumb_key, thumb_bytes, "image/jpeg")
crop_key = f"crops/{attachment_id}.jpg"
crop_sha = storage.put_object(crop_key, crop_bytes, "image/jpeg")
opt_key = f"optimized/{attachment_id}.webp"
opt_sha = storage.put_object(opt_key, opt_bytes, "image/webp")
import json
variants = [
{
"name": "thumbnail",
"content_type": "image/jpeg",
"size_bytes": len(thumb_bytes),
"width": tw,
"height": th,
"sha256": thumb_sha,
},
{
"name": "crop",
"content_type": "image/jpeg",
"size_bytes": len(crop_bytes),
"width": cw,
"height": ch,
"sha256": crop_sha,
},
{
"name": "optimized",
"content_type": "image/webp",
"size_bytes": len(opt_bytes),
"width": ow,
"height": oh,
"sha256": opt_sha,
},
]
with transaction() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE attachments
SET status = 'ready',
s3_key = %s,
variants = %s::jsonb,
error_code = NULL,
updated_at = NOW()
WHERE id = %s
""",
(final_key, json.dumps(variants), attachment_id),
)
cur.execute(
"UPDATE jobs SET status='published', lease_until=NULL, published_at=NOW() "
"WHERE job_id=%s",
(job_id,),
)
if staging_key and staging_key != final_key:
try:
storage.delete_object(staging_key)
except Exception as exc:
log.warning("staging cleanup failed: %s", exc)
log.info(
"attachment %s ready: thumbnail %dx%d, crop %dx%d, optimized %dx%d",
attachment_id, tw, th, cw, ch, ow, oh,
)
PERMANENT_ERRORS = ("invalid_image", "unsupported_media_type", "payload_too_large")
def _is_permanent(exc: Exception) -> bool:
name = type(exc).__name__.lower()
msg = str(exc).lower()
return any(p in name or p in msg for p in PERMANENT_ERRORS)
def _handler_with_retry(job: dict) -> None:
attempt = int(job.get("attempt", 1))
try:
_handle(job)
except Exception as exc:
log.exception("job %s attempt %d failed: %s", job.get("job_id"), attempt, exc)
permanent = _is_permanent(exc)
if permanent or attempt >= MAX_ATTEMPTS:
try:
broker.publish_dlq(job, str(exc))
except Exception as dlq_exc:
log.warning("DLQ publish failed: %s", dlq_exc)
with transaction() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE attachments SET status='failed', "
"error_code=%s, updated_at=NOW() WHERE id=%s",
("invalid_image" if permanent else "processing_failed",
job["attachment_id"]),
)
cur.execute(
"UPDATE jobs SET status='failed', lease_until=NULL "
"WHERE job_id=%s",
(job["job_id"],),
)
log.error(
"attachment %s failed (permanent=%s) after %d attempts",
job["attachment_id"], permanent, attempt,
)
return
delay = min(60, 2 ** attempt) + random.uniform(0, 1)
log.info("retry in %.1fs (attempt %d -> %d)", delay, attempt, attempt + 1)
with transaction() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE jobs SET status='pending', lease_until=NULL, attempt=%s "
"WHERE job_id=%s",
(attempt + 1, job["job_id"]),
)
def main() -> None:
init_pool()
storage.init_storage()
log.info("worker starting, max_attempts=%d", MAX_ATTEMPTS)
try:
broker.consume(_handler_with_retry, prefetch=int(os.environ.get("WORKER_CONCURRENCY", "2")))
finally:
close_pool()
if __name__ == "__main__":
main()