from __future__ import annotations import io import logging import os import time import random from PIL import Image, ImageOps from . import broker, storage from .db import init_pool, close_pool, transaction from .store import store, iso from opentelemetry import trace, metrics from opentelemetry.trace import SpanKind from . import otel as otel_module from .logging_config import configure_logging from prometheus_client import ( Counter as _PromCounter, Histogram as _PromHistogram, Gauge as _PromGauge, CollectorRegistry as _PromRegistry, generate_latest as _prom_generate, CONTENT_TYPE_LATEST as _PROM_CONTENT_TYPE, ) configure_logging("messenger-worker") log = logging.getLogger("messenger.worker") THUMB_SIZE = (256, 256) MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) _PROM_REGISTRY = _PromRegistry() _JOBS_PROCESSED = _PromCounter( "jobs_processed_total", "Jobs processed by outcome", ["job_type", "outcome"], registry=_PROM_REGISTRY, ) _JOB_DURATION = _PromHistogram( "job_processing_duration_seconds", "Job processing duration in seconds", ["job_type", "outcome"], buckets=(0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0), registry=_PROM_REGISTRY, ) _WORKER_IN_FLIGHT = _PromGauge( "worker_in_flight", "Jobs currently being processed by this worker", registry=_PROM_REGISTRY, ) _WORKER_CAPACITY = _PromGauge( "worker_capacity", "Worker prefetch capacity (WORKER_CONCURRENCY)", registry=_PROM_REGISTRY, ) _tracer = trace.get_tracer("messenger.worker") def _load_original(staging_key: str) -> bytes: return storage.get_object(staging_key) def _make_thumbnail(data: bytes) -> tuple[bytes, int, int]: img = Image.open(io.BytesIO(data)) img = ImageOps.exif_transpose(img) if img.mode in ("RGBA", "LA", "P"): background = Image.new("RGB", img.size, (255, 255, 255)) rgba = img.convert("RGBA") background.paste(rgba, mask=rgba.split()[-1]) img = background else: img = img.convert("RGB") img.thumbnail(THUMB_SIZE, Image.LANCZOS) w, h = img.size buf = io.BytesIO() img.save(buf, format="JPEG", quality=85, optimize=True) return buf.getvalue(), w, h def _make_crop(data: bytes, size: int = 128) -> tuple[bytes, int, int]: img = Image.open(io.BytesIO(data)) img = ImageOps.exif_transpose(img) if img.mode in ("RGBA", "LA", "P"): background = Image.new("RGB", img.size, (255, 255, 255)) rgba = img.convert("RGBA") background.paste(rgba, mask=rgba.split()[-1]) img = background else: img = img.convert("RGB") w, h = img.size side = min(w, h) left = (w - side) // 2 top = (h - side) // 2 img = img.crop((left, top, left + side, top + side)) img = img.resize((size, size), Image.LANCZOS) buf = io.BytesIO() img.save(buf, format="JPEG", quality=85, optimize=True) return buf.getvalue(), size, size def _make_optimized(data: bytes, max_side: int = 1280) -> tuple[bytes, int, int]: img = Image.open(io.BytesIO(data)) img = ImageOps.exif_transpose(img) if img.mode in ("RGBA", "LA", "P"): background = Image.new("RGB", img.size, (255, 255, 255)) rgba = img.convert("RGBA") background.paste(rgba, mask=rgba.split()[-1]) img = background else: img = img.convert("RGB") img.thumbnail((max_side, max_side), Image.LANCZOS) w, h = img.size buf = io.BytesIO() img.save(buf, format="WEBP", quality=80, method=4) return buf.getvalue(), w, h def _handle(job: dict, parent_ctx=None) -> None: attachment_id = job["attachment_id"] job_type = job.get("type", "attachment.process.v1") attempt = int(job.get("attempt", 1)) with _tracer.start_as_current_span( "job.process", context=parent_ctx, kind=SpanKind.CONSUMER, attributes={ "messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process.v1", "messaging.operation": "process", "messaging.message.id": job["job_id"], "job.type": job_type, "job.attempt": attempt, "job.attachment_id": attachment_id, }, ) as span: _WORKER_IN_FLIGHT.inc() import time as _t _start = _t.monotonic() outcome = "success" try: _handle_inner(job) except Exception as exc: outcome = "failure" span.record_exception(exc) span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc))) raise finally: elapsed = _t.monotonic() - _start _JOB_DURATION.labels(job_type=job_type, outcome=outcome).observe(elapsed) _JOBS_PROCESSED.labels(job_type=job_type, outcome=outcome).inc() _WORKER_IN_FLIGHT.dec() def _handle_inner(job: dict) -> None: attachment_id = job["attachment_id"] attempt = int(job.get("attempt", 1)) job_id = job["job_id"] log.info("processing attachment %s attempt %d (job %s)", attachment_id, attempt, job_id) with transaction() as conn: with conn.cursor() as cur: cur.execute( """ UPDATE jobs SET status = 'processing', lease_until = NOW() + INTERVAL '60 seconds' WHERE job_id = %s AND status IN ('published', 'pending') RETURNING job_id """, (job_id,), ) acquired = cur.fetchone() if acquired is None: log.info("job %s already acquired or done, skip", job_id) return with transaction() as conn: with conn.cursor() as cur: cur.execute( "UPDATE attachments SET status='processing', updated_at=NOW() " "WHERE id=%s AND status='queued'", (attachment_id,), ) cur.execute( "SELECT id, chat_id, owner_id, filename, content_type, " "size_bytes, sha256, s3_key, staging_key, status " "FROM attachments WHERE id = %s", (attachment_id,), ) row = cur.fetchone() if row is None: log.error("attachment %s not found, skip", attachment_id) return if row["status"] == "ready": log.info("attachment %s already ready, no-op", attachment_id) return staging_key = row.get("staging_key") or row["s3_key"] data = _load_original(staging_key) thumb_bytes, tw, th = _make_thumbnail(data) crop_bytes, cw, ch = _make_crop(data, 128) opt_bytes, ow, oh = _make_optimized(data, 1280) ext = row["content_type"].split("/")[-1] final_key = f"originals/{attachment_id}.{ext}" if staging_key != final_key: storage.put_object(final_key, data, row["content_type"]) thumb_key = f"thumbnails/{attachment_id}.jpg" thumb_sha = storage.put_object(thumb_key, thumb_bytes, "image/jpeg") crop_key = f"crops/{attachment_id}.jpg" crop_sha = storage.put_object(crop_key, crop_bytes, "image/jpeg") opt_key = f"optimized/{attachment_id}.webp" opt_sha = storage.put_object(opt_key, opt_bytes, "image/webp") import json variants = [ { "name": "thumbnail", "content_type": "image/jpeg", "size_bytes": len(thumb_bytes), "width": tw, "height": th, "sha256": thumb_sha, }, { "name": "crop", "content_type": "image/jpeg", "size_bytes": len(crop_bytes), "width": cw, "height": ch, "sha256": crop_sha, }, { "name": "optimized", "content_type": "image/webp", "size_bytes": len(opt_bytes), "width": ow, "height": oh, "sha256": opt_sha, }, ] with transaction() as conn: with conn.cursor() as cur: cur.execute( """ UPDATE attachments SET status = 'ready', s3_key = %s, variants = %s::jsonb, error_code = NULL, updated_at = NOW() WHERE id = %s """, (final_key, json.dumps(variants), attachment_id), ) cur.execute( "UPDATE jobs SET status='published', lease_until=NULL, published_at=NOW() " "WHERE job_id=%s", (job_id,), ) if staging_key and staging_key != final_key: try: storage.delete_object(staging_key) except Exception as exc: log.warning("staging cleanup failed: %s", exc) log.info( "attachment %s ready: thumbnail %dx%d, crop %dx%d, optimized %dx%d", attachment_id, tw, th, cw, ch, ow, oh, ) PERMANENT_ERRORS = ("invalid_image", "unsupported_media_type", "payload_too_large") def _is_permanent(exc: Exception) -> bool: name = type(exc).__name__.lower() msg = str(exc).lower() return any(p in name or p in msg for p in PERMANENT_ERRORS) def _handler_with_retry(job: dict, parent_ctx=None) -> None: attempt = int(job.get("attempt", 1)) try: _handle(job, parent_ctx=parent_ctx) except Exception as exc: log.exception("job %s attempt %d failed: %s", job.get("job_id"), attempt, exc) permanent = _is_permanent(exc) if permanent or attempt >= MAX_ATTEMPTS: try: broker.publish_dlq(job, str(exc)) except Exception as dlq_exc: log.warning("DLQ publish failed: %s", dlq_exc) with transaction() as conn: with conn.cursor() as cur: cur.execute( "UPDATE attachments SET status='failed', " "error_code=%s, updated_at=NOW() WHERE id=%s", ("invalid_image" if permanent else "processing_failed", job["attachment_id"]), ) cur.execute( "UPDATE jobs SET status='failed', lease_until=NULL " "WHERE job_id=%s", (job["job_id"],), ) log.error( "attachment %s failed (permanent=%s) after %d attempts", job["attachment_id"], permanent, attempt, ) return delay = min(60, 2 ** attempt) + random.uniform(0, 1) log.info("retry in %.1fs (attempt %d -> %d)", delay, attempt, attempt + 1) with transaction() as conn: with conn.cursor() as cur: cur.execute( "UPDATE jobs SET status='pending', lease_until=NULL, attempt=%s " "WHERE job_id=%s", (attempt + 1, job["job_id"]), ) def main() -> None: otel_module.setup_otel("messenger-worker") otel_module.instrument_libraries(psycopg=True, botocore=True, redis=False) init_pool() storage.init_storage() _WORKER_CAPACITY.set(int(os.environ.get("WORKER_CONCURRENCY", "2"))) import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer class MetricsHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == "/metrics": data = _prom_generate(_PROM_REGISTRY) self.send_response(200) self.send_header("Content-Type", _PROM_CONTENT_TYPE) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) elif self.path == "/health/live": self.send_response(200) self.send_header("Content-Type", "application/json") self.end_headers() self.wfile.write(b'{"status":"ok"}') else: self.send_response(404) self.end_headers() def log_message(self, *args): pass metrics_port = int(os.environ.get("WORKER_METRICS_PORT", "9100")) httpd = ThreadingHTTPServer(("0.0.0.0", metrics_port), MetricsHandler) threading.Thread(target=httpd.serve_forever, daemon=True, name="metrics-http").start() log.info("worker metrics on :%d/metrics", metrics_port) log.info("worker starting, max_attempts=%d", MAX_ATTEMPTS) try: broker.consume(_handler_with_retry, prefetch=int(os.environ.get("WORKER_CONCURRENCY", "2"))) finally: otel_module.shutdown_otel() close_pool() if __name__ == "__main__": main()