Files
mtusi2026lab5/app/worker.py
T
dawudukandCursor f2a3698480 Отделить обработку вложений от приёма файла.
Upload отвечает 202 после staging и записи задания, worker через RabbitMQ собирает thumbnail, crop и optimized и переживает разрыв до publish.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-09 01:08:24 +03:00

98 lines
2.7 KiB
Python

"""Separate process: consume durable jobs and publish image variants."""
from __future__ import annotations
import asyncio
import logging
import signal
import sys
from uuid import UUID
import aio_pika
from app import broker, config, db, jobs
from app.broker import payload_from
logger = logging.getLogger("messenger.worker")
def configure_logging() -> None:
logging.basicConfig(
level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
async def _settle(message: aio_pika.IncomingMessage) -> None:
try:
payload = payload_from(message.body)
job_id = UUID(payload["job_id"])
pipeline_version = int(payload["pipeline_version"])
except (ValueError, KeyError, TypeError):
logger.warning("dropping a job message that does not match the schema")
await message.ack()
return
try:
waits = 0
while True:
state = await jobs.process(job_id, pipeline_version)
if state == "busy":
waits += 1
if waits > config.JOB_LEASE_SECONDS + 5:
await message.reject(requeue=True)
return
await asyncio.sleep(1)
continue
await message.ack()
return
except Exception:
logger.warning("job settle failed job_id=%s", payload.get("job_id"))
await message.reject(requeue=True)
async def _reap_loop(stop: asyncio.Event) -> None:
while not stop.is_set():
try:
count = await jobs.reap_expired()
if count:
logger.info("requeued %s expired leases", count)
except Exception:
logger.warning("lease reaper failed")
try:
await asyncio.wait_for(stop.wait(), timeout=2)
except asyncio.TimeoutError:
continue
async def serve() -> None:
await db.connect()
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, stop.set)
except NotImplementedError:
signal.signal(sig, lambda *_: stop.set())
reaper = asyncio.create_task(_reap_loop(stop))
queue, consumer = await broker.consume(_settle, config.WORKER_CONCURRENCY)
logger.info("worker consuming prefetch=%s", config.WORKER_CONCURRENCY)
await stop.wait()
await queue.cancel(consumer)
reaper.cancel()
try:
await reaper
except asyncio.CancelledError:
pass
await broker.close()
await db.close()
def main() -> None:
configure_logging()
asyncio.run(serve())
if __name__ == "__main__":
main()