95 lines
3.7 KiB
Python
95 lines
3.7 KiB
Python
"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack."""
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass
|
|
|
|
import aio_pika
|
|
from aio_pika.abc import AbstractIncomingMessage
|
|
|
|
from app.core.jobs import JobSettings
|
|
from app.schemas.job import JobMessage
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Delivery:
|
|
id: AbstractIncomingMessage
|
|
payload: bytes
|
|
|
|
|
|
class JobBroker:
|
|
"""Один consumer/канал на экземпляр; publisher использует отдельный экземпляр."""
|
|
|
|
def __init__(self, settings: JobSettings) -> None:
|
|
self.settings = settings
|
|
self.connection = self.channel = self.queue = self.iterator = None
|
|
# Transport exceptions могут содержать URL с credentials.
|
|
logging.getLogger("aio_pika").setLevel(logging.CRITICAL)
|
|
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
|
|
|
|
async def ensure_ready(self) -> None:
|
|
if (
|
|
self.connection is None
|
|
or self.connection.is_closed
|
|
or self.channel is None
|
|
or self.channel.is_closed
|
|
):
|
|
await self.aclose()
|
|
self.connection = await aio_pika.connect(
|
|
host=self.settings.broker_host,
|
|
port=self.settings.broker_port,
|
|
login=self.settings.broker_user,
|
|
password=self.settings.broker_password_file.read_text().strip(),
|
|
virtualhost=self.settings.broker_vhost,
|
|
timeout=5,
|
|
heartbeat=30,
|
|
)
|
|
self.channel = await self.connection.channel(
|
|
publisher_confirms=True, on_return_raises=True
|
|
)
|
|
await self.channel.set_qos(prefetch_count=1)
|
|
self.queue = await self.channel.declare_queue(
|
|
self.settings.broker_queue, durable=True
|
|
)
|
|
|
|
async def publish_job(self, message: JobMessage) -> None:
|
|
await self.ensure_ready()
|
|
await self.channel.default_exchange.publish(
|
|
aio_pika.Message(
|
|
body=message.model_dump_json(exclude_none=True).encode(),
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
message_id=f"{message.job_id}:{message.attempt}",
|
|
),
|
|
routing_key=self.settings.broker_queue,
|
|
mandatory=True,
|
|
timeout=10,
|
|
)
|
|
|
|
async def receive(self, consumer: str) -> Delivery:
|
|
await self.ensure_ready()
|
|
if self.iterator is None:
|
|
self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer)
|
|
await self.iterator.__aenter__()
|
|
message = await self.iterator.__anext__()
|
|
return Delivery(message, message.body)
|
|
|
|
async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None:
|
|
await delivery_id.ack()
|
|
|
|
async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None:
|
|
# Живая чужая lease: не удерживать единственный prefetch slot навсегда.
|
|
await asyncio.sleep(self.settings.publisher_poll_seconds)
|
|
await delivery_id.nack(requeue=True)
|
|
|
|
async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None:
|
|
# Failed jobs находятся в PostgreSQL; неизвестные payload не копируются в DLQ.
|
|
logging.getLogger(__name__).warning("Rejected invalid job envelope")
|
|
await delivery_id.reject(requeue=False)
|
|
|
|
async def aclose(self) -> None:
|
|
connection = self.connection
|
|
self.connection = self.channel = self.queue = self.iterator = None
|
|
if connection is not None and not connection.is_closed:
|
|
await connection.close()
|