Files
mtusi2026lab5/app/broker.py
T
dawudukandCursor f2a3698480 Отделить обработку вложений от приёма файла.
Upload отвечает 202 после staging и записи задания, worker через RabbitMQ собирает thumbnail, crop и optimized и переживает разрыв до publish.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-09 01:08:24 +03:00

125 lines
4.1 KiB
Python

"""Durable RabbitMQ topology. Messages contain job ids, never file bytes."""
from __future__ import annotations
import asyncio
import json
import logging
import aio_pika
from app import config
logger = logging.getLogger("messenger.broker")
EXCHANGE = "attachment.jobs"
QUEUE = "attachment.jobs"
RETRY_QUEUE = "attachment.jobs.retry"
DLQ = "attachment.jobs.dlq"
ROUTING_KEY = "process"
_lock = asyncio.Lock()
_connection: aio_pika.RobustConnection | None = None
_channel: aio_pika.RobustChannel | None = None
_exchange: aio_pika.Exchange | None = None
async def _ensure() -> tuple[aio_pika.RobustChannel, aio_pika.Exchange]:
global _connection, _channel, _exchange
async with _lock:
if _channel is not None and not _channel.is_closed:
assert _exchange is not None
return _channel, _exchange
if not config.BROKER_URL:
raise RuntimeError("BROKER_URL is not set")
_connection = await aio_pika.connect_robust(config.BROKER_URL)
_channel = await _connection.channel(publisher_confirms=True)
_exchange = await _channel.declare_exchange(EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True)
await _channel.declare_queue(
RETRY_QUEUE,
durable=True,
arguments={
"x-dead-letter-exchange": EXCHANGE,
"x-dead-letter-routing-key": ROUTING_KEY,
},
)
await _channel.declare_queue(DLQ, durable=True)
queue = await _channel.declare_queue(QUEUE, durable=True)
await queue.bind(_exchange, routing_key=ROUTING_KEY)
return _channel, _exchange
async def publish_job(payload: dict) -> None:
channel, exchange = await _ensure()
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
await exchange.publish(
aio_pika.Message(
body=body,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=payload["job_id"],
),
routing_key=ROUTING_KEY,
mandatory=True,
)
async def publish_delay(payload: dict, delay_seconds: float) -> None:
channel, _exchange = await _ensure()
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
milliseconds = max(1, int(delay_seconds * 1000))
await channel.default_exchange.publish(
aio_pika.Message(
body=body,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
expiration=str(milliseconds),
message_id=payload["job_id"],
),
routing_key=RETRY_QUEUE,
mandatory=True,
)
async def publish_dlq(payload: dict) -> None:
channel, _exchange = await _ensure()
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
await channel.default_exchange.publish(
aio_pika.Message(
body=body,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=payload["job_id"],
),
routing_key=DLQ,
mandatory=True,
)
async def consume(handler, prefetch: int):
channel, _exchange = await _ensure()
await channel.set_qos(prefetch_count=max(1, prefetch))
queue = await channel.declare_queue(QUEUE, durable=True)
tag = await queue.consume(handler)
return queue, tag
async def close() -> None:
global _connection, _channel, _exchange
async with _lock:
if _connection is not None:
await _connection.close()
_connection = None
_channel = None
_exchange = None
def payload_from(raw: bytes) -> dict:
parsed = json.loads(raw.decode("utf-8"))
required = ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at")
if not isinstance(parsed, dict) or any(key not in parsed for key in required):
raise ValueError("job payload is incomplete")
if parsed["schema_version"] != 1 or parsed["type"] != "attachment.process.v1":
raise ValueError("job payload has the wrong type")
return parsed