forked from mtusi2026/mtusi2026lab5
68 lines
3.3 KiB
Python
68 lines
3.3 KiB
Python
"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений.
|
|
|
|
В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время.
|
|
Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
|
|
import aio_pika
|
|
from aio_pika import DeliveryMode, ExchangeType, Message
|
|
|
|
from . import config
|
|
from .web import log
|
|
|
|
QUEUE = "attachment.process"
|
|
DEAD_QUEUE = "attachment.process.dead"
|
|
DEAD_EXCHANGE = "attachment.dlx"
|
|
|
|
|
|
class Broker:
|
|
def __init__(self):
|
|
self.connection = None
|
|
self.channel = None
|
|
self.queue = None
|
|
self._lock = asyncio.Lock()
|
|
|
|
async def start(self, prefetch=None):
|
|
"""Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure)."""
|
|
self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT,
|
|
client_properties={"connection_name": config.SERVICE_NAME})
|
|
self.channel = await self.connection.channel(publisher_confirms=True)
|
|
if prefetch:
|
|
await self.channel.set_qos(prefetch_count=prefetch)
|
|
dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True)
|
|
dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"})
|
|
await dead.bind(dlx, routing_key="dead")
|
|
self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={
|
|
"x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"})
|
|
|
|
async def ensure(self, prefetch=None):
|
|
async with self._lock:
|
|
if self.channel is None or self.channel.is_closed:
|
|
await self.start(prefetch)
|
|
|
|
async def stop(self):
|
|
if self.connection:
|
|
try:
|
|
await asyncio.wait_for(self.connection.close(), 3)
|
|
except Exception:
|
|
pass
|
|
|
|
async def publish(self, job, headers=None):
|
|
"""Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение."""
|
|
await self.ensure()
|
|
body = json.dumps(job, separators=(",", ":")).encode("utf-8")
|
|
message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json",
|
|
message_id=job["job_id"], headers=headers or {})
|
|
await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True),
|
|
config.BROKER_TIMEOUT)
|
|
|
|
async def queue_depth(self):
|
|
await self.ensure()
|
|
q = await self.channel.declare_queue(QUEUE, durable=True, passive=True)
|
|
return q.declaration_result.message_count
|
|
|
|
async def ping(self):
|
|
return self.channel is not None and not self.channel.is_closed
|