"""RabbitMQ consumer for the version-1 attachment pipeline.""" import asyncio from hashlib import sha256 from io import BytesIO import json import logging import os import random import pickle import subprocess import sys import time from pathlib import Path from concurrent.futures import ThreadPoolExecutor from uuid import UUID, uuid4 from PIL import Image, ImageOps from .broker import QUEUE, _connection, publish_dlq from .config import load_settings from .storage.postgres import PostgresStore from .storage.s3 import S3Settings, S3Store from . import telemetry as tel from datetime import datetime, timezone logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s") class PermanentFailure(Exception): pass def fit(image, bound): longest = max(image.size) if longest <= bound: return image.copy() return image.resize((max(1, image.width * bound // longest), max(1, image.height * bound // longest)), Image.Resampling.LANCZOS) def render_isolated(data, timeout): try: result = subprocess.run([sys.executable, '-m', 'src.render'], input=data, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout) except subprocess.TimeoutExpired as error: raise TimeoutError('render deadline exceeded') from error if result.returncode in (20, 21): raise PermanentFailure('invalid_image' if result.returncode == 20 else 'image_resource_limit') if result.returncode: raise RuntimeError('renderer exited') # Only our own child process produces this payload; no external pickle input. return pickle.loads(result.stdout) def thumbnail(data: bytes) -> tuple[bytes, int, int]: with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: image = ImageOps.exif_transpose(source) if image.mode in {"RGBA", "LA"} or (image.mode == "P" and "transparency" in image.info): background = Image.new("RGBA", image.size, "white") background.alpha_composite(image.convert("RGBA")) image = background.convert("RGB") else: image = image.convert("RGB") image = fit(image, 256) output = BytesIO() image.save(output, format="JPEG", quality=85, optimize=True) return output.getvalue(), image.width, image.height def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]: """Render required variants without preserving EXIF or other input metadata.""" with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: if source.width * source.height > max_pixels: raise ValueError("image exceeds configured pixel limit") image = ImageOps.exif_transpose(source).convert("RGBA") white = Image.new("RGBA", image.size, "white") white.alpha_composite(image) rgb = white.convert("RGB") thumb = fit(rgb, 256) thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True) side = min(rgb.width, rgb.height) left, top = (rgb.width - side) // 2, (rgb.height - side) // 2 crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True) optimized = fit(rgb, 1280) optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4) return [ ("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height), ("crop", crop_out.getvalue(), "image/jpeg", 128, 128), ("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height), ] async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: # Excludes cleanup for the whole lifetime of S3 effects, including stale attempts. async with store.attachment_transaction(): return await process_locked(store, s3, payload, lease_seconds) async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: attachment_id = UUID(payload["attachment_id"]) attachment = await store.claim_attachment_job(attachment_id, lease_seconds) if attachment is None: return "skipped" payload['_claim_time'] = attachment['claim_time'] payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1)) attempt_key = uuid4().hex original_key = f"originals/{attachment_id}/{attempt_key}" await s3.copy(attachment["staging_key"], original_key, attachment["content_type"]) original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"]) rendered = await asyncio.to_thread(render_isolated, original, lease_seconds) metadata = [] for name, content, content_type, width, height in rendered: extension = "webp" if name == "optimized" else "jpg" object_key = f"variants/{attachment_id}/{attempt_key}/{name}.{extension}" await s3.put(object_key, BytesIO(content), len(content), content_type) metadata.append({"name": name, "object_key": object_key, "content_type": content_type, "size_bytes": len(content), "width": width, "height": height, "sha256": sha256(content).hexdigest()}) # Explicit, opt-in integration fault. Marker survives container restart, # so one crash exercises lease recovery without a permanent crash loop. if os.environ.get('WORKER_FAULT_AFTER_S3_PUT') == '1': marker = Path('/tmp/worker-fault-after-s3') try: with marker.open('x') as stream: stream.write(str(attachment_id)) except FileExistsError: pass else: os._exit(70) committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) if committed: try: duration = await store.ready_seconds(attachment_id) if duration is not None: tel.ready_duration.record(duration, {"job_type": "attachment"}) except Exception: tel.log.warning("telemetry.unavailable") try: await s3.delete(attachment["staging_key"]) except Exception: logging.warning('staging cleanup deferred attachment_id=%s', attachment_id) return "success" if committed else "skipped" def settle_delivery(channel, delivery_tag, future): """Only a successful durable disposition permits acknowledgement.""" try: success = future.result() is True except Exception: success = False if channel.is_open: if success: channel.basic_ack(delivery_tag) else: channel.basic_nack(delivery_tag, requeue=True) def main() -> None: tel.setup("messenger-worker") settings = load_settings() connection = _connection() channel = connection.channel() workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2"))) tel.capacity.add(workers) channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers))))) async def handle_async(payload): """Each executor thread owns its event loop and its DB/S3 clients.""" store = PostgresStore(settings.database_url) s3 = S3Store(S3Settings.load()) await store.connect() try: return await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) finally: s3.client.close() await store.close() async def schedule_retry(attachment_id, delay, claim_time, permanent_code=None): retry_store = PostgresStore(settings.database_url) await retry_store.connect() try: return await retry_store.retry_attachment_job( attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay, claim_time, permanent_code ) finally: await retry_store.close() def handle_job(body): payload = None try: payload = json.loads(body) if not isinstance(payload, dict): raise ValueError('job must be an object') if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1: raise ValueError("unexpected broker payload") UUID(payload['attachment_id']) UUID(payload['job_id']) tel.trace.get_current_span().set_attribute("messaging.retry.attempt", int(payload.get('attempt', 1))) except (ValueError, KeyError, TypeError): try: asyncio.run(publish_dlq({'failure': 'invalid_job'})) return True, "failed" except Exception: return False, "failed" try: outcome = asyncio.run(handle_async(payload)) except Exception as error: logging.warning("job failed attachment_id=%s", payload['attachment_id']) if '_claim_time' not in payload: return False, "failed" try: attachment_id = UUID(payload["attachment_id"]) attempt = int(payload.get("attempt", 1)) delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5) asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], str(error) if isinstance(error, PermanentFailure) else None)) except Exception: return False, "failed" return True, "failed" return True, outcome def handle(body, headers): started = time.monotonic() tel.busy.add(1) with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span: try: acknowledged, outcome = handle_job(body) return acknowledged except Exception: outcome = "failed" raise finally: if 'outcome' not in locals(): outcome = "failed" if outcome == "failed": span.set_status(tel.Status(tel.StatusCode.ERROR)) attrs = {"job_type": "attachment", "outcome": outcome} tel.jobs.add(1, attrs) tel.job_duration.record(time.monotonic() - started, attrs) tel.log.info("job.processed", extra={"outcome": outcome}) tel.busy.add(-1) def consume(ch, method, properties, body): future = executor.submit(handle, body, properties.headers or {}) def acknowledged(_future): if connection.is_open: connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future)) future.add_done_callback(acknowledged) try: def heartbeat(): Path('/tmp/worker-heartbeat').write_text(str(time.time())) connection.call_later(2, heartbeat) heartbeat() with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor: channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False) channel.start_consuming() finally: connection.close() if __name__ == "__main__": main()