"""Separate process: consume durable jobs and publish image variants.""" from __future__ import annotations import asyncio import logging import signal from uuid import UUID import aio_pika from app import broker, config, db, jobs, telemetry from app.broker import payload_from logger = logging.getLogger("messenger.worker") def configure_logging() -> None: telemetry.configure_logging() async def _settle(message: aio_pika.IncomingMessage) -> None: carrier = telemetry.carrier_from_headers(message.headers) parent = telemetry.remote_context(carrier.get("traceparent"), carrier.get("tracestate")) with telemetry.span( "attachment.job.consume", context=parent, kind="consumer", attributes={ "messaging.system": "rabbitmq", "messaging.destination.name": broker.QUEUE, "messaging.operation": "consume", }, ) as current: telemetry.worker_enter() try: try: payload = payload_from(message.body) job_id = UUID(payload["job_id"]) pipeline_version = int(payload["pipeline_version"]) except (ValueError, KeyError, TypeError): logger.warning("dropping a job message that does not match the schema") await message.ack() return if current is not None: current.set_attribute("job.id", str(job_id)) try: waits = 0 while True: state = await jobs.process(job_id, pipeline_version) if state == "busy": waits += 1 if waits > config.JOB_LEASE_SECONDS + 5: await message.reject(requeue=True) return await asyncio.sleep(1) continue logger.info("job settled job_id=%s state=%s", job_id, state) await message.ack() return except Exception: logger.warning("job settle failed job_id=%s", payload.get("job_id")) await message.reject(requeue=True) finally: telemetry.worker_leave() async def _reap_loop(stop: asyncio.Event) -> None: while not stop.is_set(): try: count = await jobs.reap_expired() if count: logger.info("requeued %s expired leases", count) except Exception: logger.warning("lease reaper failed") try: await asyncio.wait_for(stop.wait(), timeout=2) except asyncio.TimeoutError: continue async def serve() -> None: await db.connect() stop = asyncio.Event() loop = asyncio.get_running_loop() for sig in (signal.SIGTERM, signal.SIGINT): try: loop.add_signal_handler(sig, stop.set) except NotImplementedError: signal.signal(sig, lambda *_: stop.set()) reaper = asyncio.create_task(_reap_loop(stop)) queue, consumer = await broker.consume(_settle, config.WORKER_CONCURRENCY) logger.info("worker consuming prefetch=%s", config.WORKER_CONCURRENCY) await stop.wait() await queue.cancel(consumer) reaper.cancel() try: await reaper except asyncio.CancelledError: pass await broker.close() await db.close() telemetry.shutdown() def main() -> None: configure_logging() asyncio.run(serve()) if __name__ == "__main__": main()