"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects.""" import hashlib import json import logging import os import random import re import signal import time from contextlib import contextmanager from datetime import datetime from io import BytesIO from pathlib import Path from uuid import UUID import pika import psycopg from minio.error import S3Error from psycopg_pool import PoolTimeout from .attachments import BUCKET, storage from .db import pool from .image_variants import InvalidImage, MAX_BYTES, render from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired logger = logging.getLogger(__name__) QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) def parse_message(body: bytes) -> dict: if len(body) > 4096: raise ValueError("oversized job message") item = json.loads(body) required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} if (not isinstance(item, dict) or not required <= item.keys() or item.keys() - required - {"traceparent", "tracestate"} or type(item["schema_version"]) is not int or item["schema_version"] != 1 or item["type"] != "attachment.process.v1" or type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or type(item["attempt"]) is not int or item["attempt"] < 1 or not isinstance(item["job_id"], str) or not isinstance(item["attachment_id"], str) or not isinstance(item["created_at"], str) or ("traceparent" in item and (not isinstance(item["traceparent"], str) or re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or ("tracestate" in item and (not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))): raise ValueError("invalid job message") UUID(item["job_id"]) UUID(item["attachment_id"]) created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00")) if created_at.utcoffset() is None: raise ValueError("job timestamp lacks timezone") return item def _source_bytes(job) -> bytes: if not 0 < job["size_bytes"] <= MAX_BYTES: raise InvalidImage("invalid accepted size") # Reprocessing uses the byte-identical final original, so ready staging # can be reclaimed without making a future pipeline version impossible. key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"] response = storage().get_object(BUCKET, key) try: chunks = [] size = 0 digest = hashlib.sha256() for chunk in response.stream(64 * 1024): size += len(chunk) if size > job["size_bytes"]: raise InvalidImage("source exceeds accepted size") chunks.append(chunk) digest.update(chunk) if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]: raise InvalidImage("source checksum mismatch") return b"".join(chunks) finally: response.close() response.release_conn() def _write_variants(job, variants: dict) -> dict: extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} stored = {} for name, variant in variants.items(): key = (job["object_key"] if name == "original" else f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}") data = variant["data"] storage().put_object( BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"] ) stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"} stored[name]["object_key"] = key return stored def _delay(attempt: int) -> float: base = min(30, 2 ** min(attempt - 1, 5)) return base + random.uniform(0, base / 2) @contextmanager def _deadline(seconds: int): def expired(_signal, _frame): raise TimeoutError("image job exceeded worker deadline") previous = signal.signal(signal.SIGALRM, expired) signal.setitimer(signal.ITIMER_REAL, seconds) try: yield finally: signal.setitimer(signal.ITIMER_REAL, 0) signal.signal(signal.SIGALRM, previous) def process_message(item: dict) -> str: with pool().connection() as conn: job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS) if job is None: return "duplicate_or_stale" token = job["lease_token"] try: with _deadline(LEASE_SECONDS): data = _source_bytes(job) variants = render(data, job["content_type"]) stored = _write_variants(job, variants) with pool().connection() as conn: current = complete_job(conn, job["id"], token, stored) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) except Exception as exc: # Periodic cleanup recovers this post-COMMIT gap. logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__) return "ready" except psycopg.Error: raise except Exception as exc: if isinstance(exc, InvalidImage): code, permanent = "invalid_image", True elif isinstance(exc, TimeoutError): code, permanent = "worker_timeout", False elif isinstance(exc, S3Error) or isinstance(exc, OSError): code, permanent = "storage_unavailable", False elif isinstance(exc, ValueError): code, permanent = "processing_failed", True else: code, permanent = "processing_failed", False logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code) with pool().connection() as conn: try: return finish_attempt( conn, job["id"], token, max_attempts=MAX_ATTEMPTS, delay_seconds=_delay(job["attempt"]), error_code=code, permanent=permanent, ) except ValueError: # A lease recovery already owns the next attempt. return "stale_lease" def main() -> None: logging.basicConfig(level=logging.INFO) if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: raise SystemExit("invalid worker retry settings") url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() params = pika.URLParameters(url) params.heartbeat = max(120, LEASE_SECONDS * 2) params.socket_timeout = 5 params.blocked_connection_timeout = 10 while True: broker = None try: broker = pika.BlockingConnection(params) channel = broker.channel() channel.queue_declare(queue=QUEUE, passive=True) channel.basic_qos(prefetch_count=1) last_recovery = 0.0 while broker.is_open: if time.monotonic() - last_recovery >= 5: with pool().connection() as conn: recover_expired(conn, max_attempts=MAX_ATTEMPTS) last_recovery = time.monotonic() method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False) if method is None: broker.process_data_events(time_limit=1) continue try: item = parse_message(body) except (ValueError, TypeError, KeyError, json.JSONDecodeError): logger.warning("Rejecting invalid job message") channel.basic_reject(method.delivery_tag, requeue=False) continue try: result = process_message(item) channel.basic_ack(method.delivery_tag) logger.info("Job %s: %s", item["job_id"], result) except (psycopg.Error, PoolTimeout, OSError) as exc: logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__) channel.basic_nack(method.delivery_tag, requeue=True) time.sleep(2) except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: logger.warning("Worker reconnecting after %s", type(exc).__name__) time.sleep(2) finally: if broker is not None and broker.is_open: broker.close() if __name__ == "__main__": main()