forked from mtusi2026/mtusi2026lab5
151 lines
4.5 KiB
Python
151 lines
4.5 KiB
Python
import asyncio
|
|
import hashlib
|
|
import io
|
|
import random
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
|
|
from app.broker import consume_jobs, publish_job
|
|
from app.cache import make_cache
|
|
from app.config import config
|
|
from app.db import db
|
|
from app.s3 import s3
|
|
from app.storage import Storage
|
|
|
|
THUMB_SIZE = 256
|
|
CROP_SIZE = 128
|
|
OPTIMIZED_SIZE = 1280
|
|
|
|
_executor = ThreadPoolExecutor(max_workers=10)
|
|
|
|
PERMANENT_ERRORS = (UnidentifiedImageError,)
|
|
PERMANENT_OSERROR_MARKERS = (
|
|
"cannot identify image",
|
|
"image file is truncated",
|
|
"broken data stream",
|
|
)
|
|
|
|
|
|
def is_permanent(exc: Exception) -> bool:
|
|
if isinstance(exc, PERMANENT_ERRORS):
|
|
return True
|
|
if isinstance(exc, OSError):
|
|
text = str(exc).lower()
|
|
if any(m in text for m in PERMANENT_OSERROR_MARKERS):
|
|
return True
|
|
return False
|
|
|
|
|
|
def process_images(data: bytes):
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
img = img.convert("RGB", background=(255, 255, 255))
|
|
else:
|
|
img = img.convert("RGB")
|
|
|
|
results = {}
|
|
|
|
thumb = img.copy()
|
|
thumb.thumbnail((THUMB_SIZE, THUMB_SIZE), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
thumb.save(buf, format="JPEG", quality=85)
|
|
results["thumbnail"] = (buf.getvalue(), thumb.size[0], thumb.size[1], "image/jpeg")
|
|
|
|
crop = ImageOps.fit(img, (CROP_SIZE, CROP_SIZE), Image.LANCZOS, centering=(0.5, 0.5))
|
|
buf = io.BytesIO()
|
|
crop.save(buf, format="JPEG", quality=85)
|
|
results["crop"] = (buf.getvalue(), crop.size[0], crop.size[1], "image/jpeg")
|
|
|
|
opt = img.copy()
|
|
if opt.size[0] > OPTIMIZED_SIZE or opt.size[1] > OPTIMIZED_SIZE:
|
|
opt.thumbnail((OPTIMIZED_SIZE, OPTIMIZED_SIZE), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
opt.save(buf, format="WEBP", quality=80)
|
|
results["optimized"] = (buf.getvalue(), opt.size[0], opt.size[1], "image/webp")
|
|
|
|
return results
|
|
|
|
|
|
async def handle_job(job: dict, storage: Storage) -> None:
|
|
att_id = job.get("attachment_id")
|
|
attempt = job.get("attempt", 1)
|
|
job_id = job.get("job_id")
|
|
|
|
att = await storage.get_attachment(att_id)
|
|
if att is None:
|
|
return
|
|
if att["status"] == "ready":
|
|
return
|
|
|
|
await storage.update_attachment_status(att_id, "processing")
|
|
await storage.set_job_lease(job_id, 60)
|
|
|
|
try:
|
|
staging_key = att["_object_key"]
|
|
original = s3.get(staging_key).read()
|
|
|
|
final_key = f"originals/{att['chat_id']}/{att_id}"
|
|
s3.put(final_key, original, att["content_type"])
|
|
|
|
loop = asyncio.get_running_loop()
|
|
images = await loop.run_in_executor(_executor, process_images, original)
|
|
|
|
variants = []
|
|
variant_keys = {}
|
|
|
|
for name, (data, w, h, ctype) in images.items():
|
|
ext = "webp" if name == "optimized" else "jpg"
|
|
key = f"{name}s/{att['chat_id']}/{att_id}.{ext}"
|
|
s3.put(key, data, ctype)
|
|
variants.append({
|
|
"name": name,
|
|
"content_type": ctype,
|
|
"size_bytes": len(data),
|
|
"width": w,
|
|
"height": h,
|
|
"sha256": hashlib.sha256(data).hexdigest()
|
|
})
|
|
variant_keys[name] = key
|
|
|
|
await storage.complete_attachment_ready(att_id, final_key, variants, variant_keys)
|
|
|
|
try:
|
|
s3.remove(staging_key)
|
|
except Exception:
|
|
pass
|
|
|
|
print(f"[worker] SUCCESS {att_id}", flush=True)
|
|
except Exception as e:
|
|
print(f"[worker] ERROR {att_id}: {type(e).__name__} - {e}", flush=True)
|
|
if is_permanent(e):
|
|
await storage.fail_attachment(att_id, "invalid_image")
|
|
await publish_job(job, is_dlq=True)
|
|
return
|
|
if attempt >= config.MAX_ATTEMPTS:
|
|
await storage.fail_attachment(att_id, "retry_exhausted")
|
|
await publish_job(job, is_dlq=True)
|
|
return
|
|
new_attempt = await storage.increment_job_attempt(job_id)
|
|
job["attempt"] = new_attempt
|
|
delay = (2 ** new_attempt) + random.uniform(0, 1)
|
|
print(f"[worker] retrying in {delay:.2f} seconds...", flush=True)
|
|
await asyncio.sleep(delay)
|
|
await publish_job(job)
|
|
|
|
|
|
async def main() -> int:
|
|
await db.connect()
|
|
cache = make_cache()
|
|
storage = Storage(db.pool, cache)
|
|
s3.ensure_bucket()
|
|
print(f"[worker] started, queue={config.RABBITMQ_QUEUE}", flush=True)
|
|
await consume_jobs(lambda job: handle_job(job, storage))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main())) |