265 lines
8.7 KiB
Python
265 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import logging
|
|
import os
|
|
import time
|
|
import random
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from . import broker, storage
|
|
from .db import init_pool, close_pool, transaction
|
|
from .store import store, iso
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
log = logging.getLogger("messenger.worker")
|
|
|
|
THUMB_SIZE = (256, 256)
|
|
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
|
|
|
|
|
def _load_original(staging_key: str) -> bytes:
|
|
return storage.get_object(staging_key)
|
|
|
|
|
|
def _make_thumbnail(data: bytes) -> tuple[bytes, int, int]:
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
background = Image.new("RGB", img.size, (255, 255, 255))
|
|
rgba = img.convert("RGBA")
|
|
background.paste(rgba, mask=rgba.split()[-1])
|
|
img = background
|
|
else:
|
|
img = img.convert("RGB")
|
|
img.thumbnail(THUMB_SIZE, Image.LANCZOS)
|
|
w, h = img.size
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=85, optimize=True)
|
|
return buf.getvalue(), w, h
|
|
|
|
def _make_crop(data: bytes, size: int = 128) -> tuple[bytes, int, int]:
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
background = Image.new("RGB", img.size, (255, 255, 255))
|
|
rgba = img.convert("RGBA")
|
|
background.paste(rgba, mask=rgba.split()[-1])
|
|
img = background
|
|
else:
|
|
img = img.convert("RGB")
|
|
w, h = img.size
|
|
side = min(w, h)
|
|
left = (w - side) // 2
|
|
top = (h - side) // 2
|
|
img = img.crop((left, top, left + side, top + side))
|
|
img = img.resize((size, size), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=85, optimize=True)
|
|
return buf.getvalue(), size, size
|
|
|
|
|
|
def _make_optimized(data: bytes, max_side: int = 1280) -> tuple[bytes, int, int]:
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
background = Image.new("RGB", img.size, (255, 255, 255))
|
|
rgba = img.convert("RGBA")
|
|
background.paste(rgba, mask=rgba.split()[-1])
|
|
img = background
|
|
else:
|
|
img = img.convert("RGB")
|
|
img.thumbnail((max_side, max_side), Image.LANCZOS)
|
|
w, h = img.size
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="WEBP", quality=80, method=4)
|
|
return buf.getvalue(), w, h
|
|
|
|
def _handle(job: dict) -> None:
|
|
attachment_id = job["attachment_id"]
|
|
attempt = int(job.get("attempt", 1))
|
|
job_id = job["job_id"]
|
|
log.info("processing attachment %s attempt %d (job %s)", attachment_id, attempt, job_id)
|
|
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'processing',
|
|
lease_until = NOW() + INTERVAL '60 seconds'
|
|
WHERE job_id = %s AND status IN ('published', 'pending')
|
|
RETURNING job_id
|
|
""",
|
|
(job_id,),
|
|
)
|
|
acquired = cur.fetchone()
|
|
if acquired is None:
|
|
log.info("job %s already acquired or done, skip", job_id)
|
|
return
|
|
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE attachments SET status='processing', updated_at=NOW() "
|
|
"WHERE id=%s AND status='queued'",
|
|
(attachment_id,),
|
|
)
|
|
cur.execute(
|
|
"SELECT id, chat_id, owner_id, filename, content_type, "
|
|
"size_bytes, sha256, s3_key, staging_key, status "
|
|
"FROM attachments WHERE id = %s",
|
|
(attachment_id,),
|
|
)
|
|
row = cur.fetchone()
|
|
if row is None:
|
|
log.error("attachment %s not found, skip", attachment_id)
|
|
return
|
|
if row["status"] == "ready":
|
|
log.info("attachment %s already ready, no-op", attachment_id)
|
|
return
|
|
|
|
staging_key = row.get("staging_key") or row["s3_key"]
|
|
|
|
data = _load_original(staging_key)
|
|
|
|
thumb_bytes, tw, th = _make_thumbnail(data)
|
|
crop_bytes, cw, ch = _make_crop(data, 128)
|
|
opt_bytes, ow, oh = _make_optimized(data, 1280)
|
|
|
|
ext = row["content_type"].split("/")[-1]
|
|
final_key = f"originals/{attachment_id}.{ext}"
|
|
if staging_key != final_key:
|
|
storage.put_object(final_key, data, row["content_type"])
|
|
|
|
thumb_key = f"thumbnails/{attachment_id}.jpg"
|
|
thumb_sha = storage.put_object(thumb_key, thumb_bytes, "image/jpeg")
|
|
|
|
crop_key = f"crops/{attachment_id}.jpg"
|
|
crop_sha = storage.put_object(crop_key, crop_bytes, "image/jpeg")
|
|
|
|
opt_key = f"optimized/{attachment_id}.webp"
|
|
opt_sha = storage.put_object(opt_key, opt_bytes, "image/webp")
|
|
|
|
import json
|
|
variants = [
|
|
{
|
|
"name": "thumbnail",
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": len(thumb_bytes),
|
|
"width": tw,
|
|
"height": th,
|
|
"sha256": thumb_sha,
|
|
},
|
|
{
|
|
"name": "crop",
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": len(crop_bytes),
|
|
"width": cw,
|
|
"height": ch,
|
|
"sha256": crop_sha,
|
|
},
|
|
{
|
|
"name": "optimized",
|
|
"content_type": "image/webp",
|
|
"size_bytes": len(opt_bytes),
|
|
"width": ow,
|
|
"height": oh,
|
|
"sha256": opt_sha,
|
|
},
|
|
]
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready',
|
|
s3_key = %s,
|
|
variants = %s::jsonb,
|
|
error_code = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(final_key, json.dumps(variants), attachment_id),
|
|
)
|
|
cur.execute(
|
|
"UPDATE jobs SET status='published', lease_until=NULL, published_at=NOW() "
|
|
"WHERE job_id=%s",
|
|
(job_id,),
|
|
)
|
|
|
|
if staging_key and staging_key != final_key:
|
|
try:
|
|
storage.delete_object(staging_key)
|
|
except Exception as exc:
|
|
log.warning("staging cleanup failed: %s", exc)
|
|
|
|
log.info(
|
|
"attachment %s ready: thumbnail %dx%d, crop %dx%d, optimized %dx%d",
|
|
attachment_id, tw, th, cw, ch, ow, oh,
|
|
)
|
|
|
|
|
|
PERMANENT_ERRORS = ("invalid_image", "unsupported_media_type", "payload_too_large")
|
|
|
|
|
|
def _is_permanent(exc: Exception) -> bool:
|
|
name = type(exc).__name__.lower()
|
|
msg = str(exc).lower()
|
|
return any(p in name or p in msg for p in PERMANENT_ERRORS)
|
|
|
|
|
|
def _handler_with_retry(job: dict) -> None:
|
|
attempt = int(job.get("attempt", 1))
|
|
try:
|
|
_handle(job)
|
|
except Exception as exc:
|
|
log.exception("job %s attempt %d failed: %s", job.get("job_id"), attempt, exc)
|
|
permanent = _is_permanent(exc)
|
|
if permanent or attempt >= MAX_ATTEMPTS:
|
|
try:
|
|
broker.publish_dlq(job, str(exc))
|
|
except Exception as dlq_exc:
|
|
log.warning("DLQ publish failed: %s", dlq_exc)
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE attachments SET status='failed', "
|
|
"error_code=%s, updated_at=NOW() WHERE id=%s",
|
|
("invalid_image" if permanent else "processing_failed",
|
|
job["attachment_id"]),
|
|
)
|
|
cur.execute(
|
|
"UPDATE jobs SET status='failed', lease_until=NULL "
|
|
"WHERE job_id=%s",
|
|
(job["job_id"],),
|
|
)
|
|
log.error(
|
|
"attachment %s failed (permanent=%s) after %d attempts",
|
|
job["attachment_id"], permanent, attempt,
|
|
)
|
|
return
|
|
delay = min(60, 2 ** attempt) + random.uniform(0, 1)
|
|
log.info("retry in %.1fs (attempt %d -> %d)", delay, attempt, attempt + 1)
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE jobs SET status='pending', lease_until=NULL, attempt=%s "
|
|
"WHERE job_id=%s",
|
|
(attempt + 1, job["job_id"]),
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
init_pool()
|
|
storage.init_storage()
|
|
log.info("worker starting, max_attempts=%d", MAX_ATTEMPTS)
|
|
try:
|
|
broker.consume(_handler_with_retry, prefetch=int(os.environ.get("WORKER_CONCURRENCY", "2")))
|
|
finally:
|
|
close_pool()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |