forked from mtusi2026/mtusi2026lab5
Upload отвечает 202 после staging и записи задания, worker через RabbitMQ собирает thumbnail, crop и optimized и переживает разрыв до publish. Co-authored-by: Cursor <cursoragent@cursor.com>
125 lines
4.1 KiB
Python
125 lines
4.1 KiB
Python
"""Durable RabbitMQ topology. Messages contain job ids, never file bytes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
|
|
import aio_pika
|
|
|
|
from app import config
|
|
|
|
logger = logging.getLogger("messenger.broker")
|
|
|
|
EXCHANGE = "attachment.jobs"
|
|
QUEUE = "attachment.jobs"
|
|
RETRY_QUEUE = "attachment.jobs.retry"
|
|
DLQ = "attachment.jobs.dlq"
|
|
ROUTING_KEY = "process"
|
|
|
|
_lock = asyncio.Lock()
|
|
_connection: aio_pika.RobustConnection | None = None
|
|
_channel: aio_pika.RobustChannel | None = None
|
|
_exchange: aio_pika.Exchange | None = None
|
|
|
|
|
|
async def _ensure() -> tuple[aio_pika.RobustChannel, aio_pika.Exchange]:
|
|
global _connection, _channel, _exchange
|
|
async with _lock:
|
|
if _channel is not None and not _channel.is_closed:
|
|
assert _exchange is not None
|
|
return _channel, _exchange
|
|
if not config.BROKER_URL:
|
|
raise RuntimeError("BROKER_URL is not set")
|
|
_connection = await aio_pika.connect_robust(config.BROKER_URL)
|
|
_channel = await _connection.channel(publisher_confirms=True)
|
|
_exchange = await _channel.declare_exchange(EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True)
|
|
await _channel.declare_queue(
|
|
RETRY_QUEUE,
|
|
durable=True,
|
|
arguments={
|
|
"x-dead-letter-exchange": EXCHANGE,
|
|
"x-dead-letter-routing-key": ROUTING_KEY,
|
|
},
|
|
)
|
|
await _channel.declare_queue(DLQ, durable=True)
|
|
queue = await _channel.declare_queue(QUEUE, durable=True)
|
|
await queue.bind(_exchange, routing_key=ROUTING_KEY)
|
|
return _channel, _exchange
|
|
|
|
|
|
async def publish_job(payload: dict) -> None:
|
|
channel, exchange = await _ensure()
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
await exchange.publish(
|
|
aio_pika.Message(
|
|
body=body,
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
message_id=payload["job_id"],
|
|
),
|
|
routing_key=ROUTING_KEY,
|
|
mandatory=True,
|
|
)
|
|
|
|
|
|
async def publish_delay(payload: dict, delay_seconds: float) -> None:
|
|
channel, _exchange = await _ensure()
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
milliseconds = max(1, int(delay_seconds * 1000))
|
|
await channel.default_exchange.publish(
|
|
aio_pika.Message(
|
|
body=body,
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
expiration=str(milliseconds),
|
|
message_id=payload["job_id"],
|
|
),
|
|
routing_key=RETRY_QUEUE,
|
|
mandatory=True,
|
|
)
|
|
|
|
|
|
async def publish_dlq(payload: dict) -> None:
|
|
channel, _exchange = await _ensure()
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
await channel.default_exchange.publish(
|
|
aio_pika.Message(
|
|
body=body,
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
message_id=payload["job_id"],
|
|
),
|
|
routing_key=DLQ,
|
|
mandatory=True,
|
|
)
|
|
|
|
|
|
async def consume(handler, prefetch: int):
|
|
channel, _exchange = await _ensure()
|
|
await channel.set_qos(prefetch_count=max(1, prefetch))
|
|
queue = await channel.declare_queue(QUEUE, durable=True)
|
|
tag = await queue.consume(handler)
|
|
return queue, tag
|
|
|
|
|
|
async def close() -> None:
|
|
global _connection, _channel, _exchange
|
|
async with _lock:
|
|
if _connection is not None:
|
|
await _connection.close()
|
|
_connection = None
|
|
_channel = None
|
|
_exchange = None
|
|
|
|
|
|
def payload_from(raw: bytes) -> dict:
|
|
parsed = json.loads(raw.decode("utf-8"))
|
|
required = ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at")
|
|
if not isinstance(parsed, dict) or any(key not in parsed for key in required):
|
|
raise ValueError("job payload is incomplete")
|
|
if parsed["schema_version"] != 1 or parsed["type"] != "attachment.process.v1":
|
|
raise ValueError("job payload has the wrong type")
|
|
return parsed
|