Files
mtusi2026lab5/app/worker.py
T

151 lines
4.5 KiB
Python

import asyncio
import hashlib
import io
import random
import sys
from concurrent.futures import ThreadPoolExecutor
from PIL import Image, ImageOps, UnidentifiedImageError
from app.broker import consume_jobs, publish_job
from app.cache import make_cache
from app.config import config
from app.db import db
from app.s3 import s3
from app.storage import Storage
THUMB_SIZE = 256
CROP_SIZE = 128
OPTIMIZED_SIZE = 1280
_executor = ThreadPoolExecutor(max_workers=10)
PERMANENT_ERRORS = (UnidentifiedImageError,)
PERMANENT_OSERROR_MARKERS = (
"cannot identify image",
"image file is truncated",
"broken data stream",
)
def is_permanent(exc: Exception) -> bool:
if isinstance(exc, PERMANENT_ERRORS):
return True
if isinstance(exc, OSError):
text = str(exc).lower()
if any(m in text for m in PERMANENT_OSERROR_MARKERS):
return True
return False
def process_images(data: bytes):
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
img = img.convert("RGB", background=(255, 255, 255))
else:
img = img.convert("RGB")
results = {}
thumb = img.copy()
thumb.thumbnail((THUMB_SIZE, THUMB_SIZE), Image.LANCZOS)
buf = io.BytesIO()
thumb.save(buf, format="JPEG", quality=85)
results["thumbnail"] = (buf.getvalue(), thumb.size[0], thumb.size[1], "image/jpeg")
crop = ImageOps.fit(img, (CROP_SIZE, CROP_SIZE), Image.LANCZOS, centering=(0.5, 0.5))
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=85)
results["crop"] = (buf.getvalue(), crop.size[0], crop.size[1], "image/jpeg")
opt = img.copy()
if opt.size[0] > OPTIMIZED_SIZE or opt.size[1] > OPTIMIZED_SIZE:
opt.thumbnail((OPTIMIZED_SIZE, OPTIMIZED_SIZE), Image.LANCZOS)
buf = io.BytesIO()
opt.save(buf, format="WEBP", quality=80)
results["optimized"] = (buf.getvalue(), opt.size[0], opt.size[1], "image/webp")
return results
async def handle_job(job: dict, storage: Storage) -> None:
att_id = job.get("attachment_id")
attempt = job.get("attempt", 1)
job_id = job.get("job_id")
att = await storage.get_attachment(att_id)
if att is None:
return
if att["status"] == "ready":
return
await storage.update_attachment_status(att_id, "processing")
await storage.set_job_lease(job_id, 60)
try:
staging_key = att["_object_key"]
original = s3.get(staging_key).read()
final_key = f"originals/{att['chat_id']}/{att_id}"
s3.put(final_key, original, att["content_type"])
loop = asyncio.get_running_loop()
images = await loop.run_in_executor(_executor, process_images, original)
variants = []
variant_keys = {}
for name, (data, w, h, ctype) in images.items():
ext = "webp" if name == "optimized" else "jpg"
key = f"{name}s/{att['chat_id']}/{att_id}.{ext}"
s3.put(key, data, ctype)
variants.append({
"name": name,
"content_type": ctype,
"size_bytes": len(data),
"width": w,
"height": h,
"sha256": hashlib.sha256(data).hexdigest()
})
variant_keys[name] = key
await storage.complete_attachment_ready(att_id, final_key, variants, variant_keys)
try:
s3.remove(staging_key)
except Exception:
pass
print(f"[worker] SUCCESS {att_id}", flush=True)
except Exception as e:
print(f"[worker] ERROR {att_id}: {type(e).__name__} - {e}", flush=True)
if is_permanent(e):
await storage.fail_attachment(att_id, "invalid_image")
await publish_job(job, is_dlq=True)
return
if attempt >= config.MAX_ATTEMPTS:
await storage.fail_attachment(att_id, "retry_exhausted")
await publish_job(job, is_dlq=True)
return
new_attempt = await storage.increment_job_attempt(job_id)
job["attempt"] = new_attempt
delay = (2 ** new_attempt) + random.uniform(0, 1)
print(f"[worker] retrying in {delay:.2f} seconds...", flush=True)
await asyncio.sleep(delay)
await publish_job(job)
async def main() -> int:
await db.connect()
cache = make_cache()
storage = Storage(db.pool, cache)
s3.ensure_bucket()
print(f"[worker] started, queue={config.RABBITMQ_QUEUE}", flush=True)
await consume_jobs(lambda job: handle_job(job, storage))
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))