"""RabbitMQ consumer process for durable image processing.""" import asyncio import json import logging import random import socket import threading import uuid import time from datetime import datetime, timedelta, timezone import aio_pika from aio_pika import DeliveryMode, Message from botocore.exceptions import BotoCoreError, ClientError from sqlalchemy import select, update from sqlalchemy.exc import SQLAlchemyError from app.core.config import settings from app.core.tls import service_ssl_context from app.db.models import Attachment, AttachmentVariant, ProcessingJob from app.db.session import SessionLocal from app.services.image_processor import ( ImageProcessingTimeout, create_variants_with_timeout, ) from app.services.object_storage import client as s3_client from app.core.worker_telemetry import ( process_job_with_telemetry, setup_worker_telemetry, ) logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s", ) logger = logging.getLogger("attachment-worker") WORKER_ID = ( f"{socket.gethostname()[:40]}-{uuid.uuid4().hex[:16]}" ) DEAD_ROUTING_KEY = f"{settings.rabbitmq_queue}.dead" PROCESSING_SEMAPHORE = None def utc_now() -> datetime: return datetime.now(timezone.utc) def _safe_uuid(value) -> uuid.UUID: return uuid.UUID(str(value)) def _claim_job(job_id, attachment_id, pipeline_version) -> dict: """Atomically claim a job or return the reason it cannot be claimed.""" job_id = _safe_uuid(job_id) attachment_id = _safe_uuid(attachment_id) pipeline_version = int(pipeline_version) db = SessionLocal() try: job = db.scalar( select(ProcessingJob) .where( ProcessingJob.id == job_id, ProcessingJob.attachment_id == attachment_id, ProcessingJob.pipeline_version == pipeline_version, ) .with_for_update() ) if job is None: db.rollback() return {"state": "missing"} attachment = db.get(Attachment, attachment_id) if attachment is None: db.rollback() return {"state": "missing"} now = utc_now() if attachment.status == "ready": db.commit() return {"state": "ready"} if job.status == "failed": db.commit() return {"state": "failed"} if ( job.status == "processing" and job.lease_until is not None and job.lease_until > now ): delay = (job.lease_until - now).total_seconds() + 1 db.commit() return { "state": "busy", "delay": max(1.0, delay), } if job.available_at > now: delay = (job.available_at - now).total_seconds() + 0.5 db.commit() return { "state": "deferred", "delay": max(1.0, delay), } if job.attempt >= settings.worker_max_attempts: job.status = "failed" job.worker_id = None job.lease_until = None job.last_error_code = "processing_attempts_exhausted" attachment.status = "failed" attachment.error_code = "processing_attempts_exhausted" db.commit() return {"state": "failed"} claim_worker_id = f"{WORKER_ID}:{uuid.uuid4().hex[:16]}" job.status = "processing" job.worker_id = claim_worker_id job.lease_until = now + timedelta( seconds=settings.worker_lease_seconds ) job.attempt += 1 attachment.status = "processing" attachment.error_code = None claimed_attempt = job.attempt queue_lag_ms = max( 0.0, (now - job.available_at).total_seconds() * 1000, ) db.commit() logger.info( "Claimed job=%s attachment=%s attempt=%s queue_lag_ms=%.2f", job_id, attachment_id, claimed_attempt, queue_lag_ms, ) return { "state": "claimed", "worker_id": claim_worker_id, "attempt": claimed_attempt, "queue_lag_ms": queue_lag_ms, } except Exception: db.rollback() raise finally: db.close() def _renew_lease( job_id: uuid.UUID, worker_id: str, attempt: int, stop_event: threading.Event, lost_event: threading.Event, ) -> None: """Renew the lease while image processing is running.""" interval = max( 5, settings.worker_lease_seconds // 3, ) while not stop_event.wait(interval): db = SessionLocal() try: result = db.execute( update(ProcessingJob) .where( ProcessingJob.id == job_id, ProcessingJob.worker_id == worker_id, ProcessingJob.attempt == attempt, ProcessingJob.status == "processing", ) .values( lease_until=utc_now() + timedelta(seconds=settings.worker_lease_seconds) ) ) db.commit() if result.rowcount != 1: lost_event.set() logger.warning( "Lease ownership lost: job=%s worker=%s", job_id, worker_id, ) return except Exception: db.rollback() lost_event.set() logger.exception( "Could not renew lease for job=%s", job_id, ) return finally: db.close() def _fail_job( job_id, attachment_id, worker_id, attempt, error_code, permanent=False, ) -> dict: """Record a failure only if this worker still owns the job.""" db = SessionLocal() try: job = db.scalar( select(ProcessingJob) .where(ProcessingJob.id == job_id) .with_for_update() ) attachment = db.scalar( select(Attachment) .where(Attachment.id == attachment_id) .with_for_update() ) if job is None or attachment is None: db.rollback() return {"state": "missing"} if ( job.status != "processing" or job.worker_id != worker_id or job.attempt != attempt ): db.rollback() logger.warning( "Ignoring failure from stale worker: job=%s worker=%s", job_id, worker_id, ) return { "state": "stale", "delay": float(settings.worker_lease_seconds), } exhausted = job.attempt >= settings.worker_max_attempts job.last_error_code = error_code if permanent or exhausted: job.status = "failed" job.worker_id = None job.lease_until = None attachment.status = "failed" attachment.error_code = error_code db.commit() logger.error( "Job permanently failed: job=%s error=%s", job_id, error_code, ) return {"state": "failed"} delay = min( settings.worker_retry_base_seconds * (2 ** max(job.attempt - 1, 0)), settings.worker_retry_max_seconds, ) delay += random.uniform(0, delay * 0.25) job.status = "queued" job.available_at = utc_now() + timedelta(seconds=delay) job.lease_until = None job.worker_id = None attachment.status = "queued" attachment.error_code = error_code db.commit() logger.warning( "Job scheduled for retry: job=%s delay=%.2fs error=%s", job_id, delay, error_code, ) return { "state": "retry", "delay": delay, } except Exception: db.rollback() raise finally: db.close() def _process_impl(payload: dict) -> dict: """Process one image using a fenced, renewable job lease.""" job_id = _safe_uuid(payload["job_id"]) attachment_id = _safe_uuid(payload["attachment_id"]) pipeline_version = int(payload["pipeline_version"]) claim = _claim_job( job_id, attachment_id, pipeline_version, ) if claim["state"] != "claimed": return claim worker_id = claim["worker_id"] attempt = claim["attempt"] queue_lag_ms = claim["queue_lag_ms"] processing_started_at = time.perf_counter() stop_event = threading.Event() lost_event = threading.Event() heartbeat = threading.Thread( target=_renew_lease, args=( job_id, worker_id, attempt, stop_event, lost_event, ), daemon=True, ) heartbeat.start() staging_key = None db = None try: db = SessionLocal() attachment = db.get(Attachment, attachment_id) if attachment is None: raise ValueError("attachment_missing") staging_key = attachment.staging_key object_key = attachment.object_key content_type = attachment.content_type if not staging_key: raise ValueError("staging_missing") db.close() db = None if lost_event.is_set(): return { "state": "retry", "delay": float(settings.worker_lease_seconds), } s3 = s3_client() response = s3.get_object( Bucket=settings.s3_bucket, Key=staging_key, ) body = response["Body"] try: attachment_data = body.read( settings.max_upload_bytes + 1 ) finally: body.close() if len(attachment_data) > settings.max_upload_bytes: raise ValueError("image_too_large") if lost_event.is_set(): return { "state": "retry", "delay": float(settings.worker_lease_seconds), } variants = create_variants_with_timeout( attachment_data, settings.image_processing_timeout_seconds, ) # The original object uses a deterministic key. s3.put_object( Bucket=settings.s3_bucket, Key=object_key, Body=attachment_data, ContentType=content_type, ) variant_records = [] for variant in variants: key = ( f"variants/{attachment_id}/" f"v{pipeline_version}/" f"{variant.name}.{variant.extension}" ) s3.put_object( Bucket=settings.s3_bucket, Key=key, Body=variant.data, ContentType=variant.content_type, ) variant_records.append( { "name": variant.name, "key": key, "content_type": variant.content_type, "width": variant.width, "height": variant.height, "size_bytes": len(variant.data), "sha256": variant.sha256, } ) if lost_event.is_set(): logger.warning( "Worker lost lease before final commit: job=%s", job_id, ) return { "state": "retry", "delay": float(settings.worker_lease_seconds), } # Use a fresh transaction and verify the fencing token. db = SessionLocal() job = db.scalar( select(ProcessingJob) .where(ProcessingJob.id == job_id) .with_for_update() ) attachment = db.scalar( select(Attachment) .where(Attachment.id == attachment_id) .with_for_update() ) if job is None or attachment is None: db.rollback() return {"state": "missing"} if ( job.status != "processing" or job.worker_id != worker_id or job.attempt != attempt or job.lease_until is None or job.lease_until <= utc_now() or lost_event.is_set() ): db.rollback() logger.warning( "Fencing check rejected stale worker: job=%s", job_id, ) return { "state": "retry", "delay": float(settings.worker_lease_seconds), } for item in variant_records: existing = db.scalar( select(AttachmentVariant) .where( AttachmentVariant.attachment_id == attachment_id, AttachmentVariant.variant == item["name"], ) ) if existing is None: existing = AttachmentVariant( attachment_id=attachment_id, variant=item["name"], object_key=item["key"], content_type=item["content_type"], width=item["width"], height=item["height"], size_bytes=item["size_bytes"], sha256=item["sha256"], ) db.add(existing) else: existing.object_key = item["key"] existing.content_type = item["content_type"] existing.width = item["width"] existing.height = item["height"] existing.size_bytes = item["size_bytes"] existing.sha256 = item["sha256"] job.status = "ready" job.lease_until = None job.worker_id = None job.last_error_code = None attachment.status = "ready" attachment.error_code = None attachment.updated_at = utc_now() db.commit() processing_elapsed_ms = ( time.perf_counter() - processing_started_at ) * 1000 logger.info( ( "Image processing completed: attachment=%s " "attempt=%s queue_lag_ms=%.2f processing_elapsed_ms=%.2f" ), attachment_id, attempt, queue_lag_ms, processing_elapsed_ms, ) db.close() db = None # Staging cleanup is performed only after the final DB commit. try: s3.delete_object( Bucket=settings.s3_bucket, Key=staging_key, ) cleanup_db = SessionLocal() try: cleanup_attachment = cleanup_db.scalar( select(Attachment) .where(Attachment.id == attachment_id) .with_for_update() ) if ( cleanup_attachment is not None and cleanup_attachment.status == "ready" and cleanup_attachment.staging_key == staging_key ): cleanup_attachment.staging_key = None cleanup_db.commit() else: cleanup_db.rollback() finally: cleanup_db.close() except Exception: logger.warning( "Could not delete staging object for attachment=%s", attachment_id, exc_info=True, ) return {"state": "ready"} except ImageProcessingTimeout: if db is not None: db.rollback() stop_event.set() heartbeat.join(timeout=2) logger.warning( "Image processing timed out: job=%s attachment=%s timeout=%ss", job_id, attachment_id, settings.image_processing_timeout_seconds, ) return _fail_job( job_id, attachment_id, worker_id, attempt, "image_processing_timeout", permanent=False, ) except ValueError as exc: if db is not None: db.rollback() stop_event.set() heartbeat.join(timeout=2) code = str(exc)[:64] or "invalid_image" return _fail_job( job_id, attachment_id, worker_id, attempt, code, permanent=True, ) except ClientError as exc: if db is not None: db.rollback() stop_event.set() heartbeat.join(timeout=2) error = exc.response.get("Error", {}) error_code = error.get("Code", "") if error_code in { "NoSuchKey", "NoSuchBucket", "404", "NotFound", }: return _fail_job( job_id, attachment_id, worker_id, attempt, "staging_missing", permanent=True, ) return _fail_job( job_id, attachment_id, worker_id, attempt, "storage_unavailable", permanent=False, ) except (BotoCoreError, SQLAlchemyError): if db is not None: db.rollback() stop_event.set() heartbeat.join(timeout=2) return _fail_job( job_id, attachment_id, worker_id, attempt, "processing_unavailable", permanent=False, ) except Exception: if db is not None: db.rollback() logger.exception( "Unexpected processing error: job=%s", job_id, ) stop_event.set() heartbeat.join(timeout=2) return _fail_job( job_id, attachment_id, worker_id, attempt, "processing_failed", permanent=False, ) finally: stop_event.set() heartbeat.join(timeout=2) if db is not None: db.close() def _process(payload: dict) -> dict: return process_job_with_telemetry( _process_impl, payload, ) async def _publish_delayed( exchange, payload: dict, delay_seconds: float, ) -> None: """Publish a persistent message to the TTL retry queue.""" delay_ms = max( 1000, int(delay_seconds * 1000), ) message = Message( body=json.dumps( payload, separators=(",", ":"), ).encode("utf-8"), content_type="application/json", delivery_mode=DeliveryMode.PERSISTENT, message_id=str(uuid.uuid4()), type="attachment.process.retry", expiration=timedelta(milliseconds=delay_ms), ) await exchange.publish( message, routing_key=f"{settings.rabbitmq_queue}.retry", mandatory=True, ) logger.info( "Scheduled delayed delivery in %.2fs", delay_ms / 1000, ) async def _quarantine_message(exchange, message) -> None: """Move an invalid message to the durable dead-letter queue.""" dead_message = Message( body=message.body, content_type=message.content_type or "application/octet-stream", delivery_mode=DeliveryMode.PERSISTENT, message_id=message.message_id or str(uuid.uuid4()), type="attachment.process.invalid", ) await exchange.publish( dead_message, routing_key=DEAD_ROUTING_KEY, mandatory=True, ) await message.ack() logger.error( "Invalid message moved to dead-letter queue: message_id=%s", message.message_id, ) async def handle_message( message: aio_pika.IncomingMessage, retry_exchange, dead_exchange, semaphore: asyncio.Semaphore, ) -> None: """Consume, process, retry or quarantine one RabbitMQ message.""" async with semaphore: try: payload = json.loads(message.body.decode("utf-8")) if not isinstance(payload, dict): raise ValueError("invalid_payload") for field in ( "job_id", "attachment_id", "pipeline_version", ): if field not in payload: raise ValueError("missing_payload_field") _safe_uuid(payload["job_id"]) _safe_uuid(payload["attachment_id"]) int(payload["pipeline_version"]) except (ValueError, TypeError, KeyError, UnicodeDecodeError): try: await _quarantine_message( dead_exchange, message, ) except Exception: logger.exception( "Could not quarantine invalid message" ) await message.nack(requeue=True) return try: result = await asyncio.to_thread( _process, payload, ) state = result["state"] if state in { "busy", "deferred", "retry", }: await _publish_delayed( retry_exchange, payload, result.get("delay", 1.0), ) await message.ack() logger.info( "Message handled: job=%s state=%s", payload["job_id"], state, ) except asyncio.CancelledError: raise except Exception: logger.exception( "Message processing failed: message_id=%s", message.message_id, ) # If delayed publication or DB processing fails, # preserve the original message. await message.nack(requeue=True) async def main() -> None: """Start the durable RabbitMQ consumer.""" setup_worker_telemetry() connection = await aio_pika.connect_robust( settings.rabbitmq_url, ssl=settings.rabbitmq_url.startswith("amqps://"), ssl_context=service_ssl_context() if settings.rabbitmq_url.startswith("amqps://") else None, ) async with connection: channel = await connection.channel( publisher_confirms=True ) await channel.set_qos( prefetch_count=settings.rabbitmq_prefetch ) exchange = await channel.declare_exchange( settings.rabbitmq_exchange, aio_pika.ExchangeType.DIRECT, durable=True, ) queue = await channel.declare_queue( settings.rabbitmq_queue, durable=True, ) await queue.bind( exchange, routing_key=settings.rabbitmq_queue, ) # Delayed retry exchange and queue. retry_exchange = await channel.declare_exchange( f"{settings.rabbitmq_exchange}.retry", aio_pika.ExchangeType.DIRECT, durable=True, ) retry_queue = await channel.declare_queue( f"{settings.rabbitmq_queue}.retry", durable=True, arguments={ "x-dead-letter-exchange": settings.rabbitmq_exchange, "x-dead-letter-routing-key": settings.rabbitmq_queue, }, ) await retry_queue.bind( retry_exchange, routing_key=f"{settings.rabbitmq_queue}.retry", ) # Durable quarantine queue for invalid messages. dead_exchange = await channel.declare_exchange( f"{settings.rabbitmq_exchange}.dead", aio_pika.ExchangeType.DIRECT, durable=True, ) dead_queue = await channel.declare_queue( f"{settings.rabbitmq_queue}.dead", durable=True, ) await dead_queue.bind( dead_exchange, routing_key=DEAD_ROUTING_KEY, ) semaphore = asyncio.Semaphore( settings.worker_concurrency ) async def consumer(message): await handle_message( message, retry_exchange, dead_exchange, semaphore, ) logger.info( "Worker %s consuming queue=%s concurrency=%s prefetch=%s", WORKER_ID, settings.rabbitmq_queue, settings.worker_concurrency, settings.rabbitmq_prefetch, ) await queue.consume(consumer) await asyncio.Future() if __name__ == "__main__": asyncio.run(main())