forked from mtusi2026/mtusi2026lab6
API и worker отдают метрики, логи и трассы. Правила SLO записаны, прогон инцидентов ещё впереди.
139 lines
4.7 KiB
Python
139 lines
4.7 KiB
Python
"""Durable RabbitMQ topology. Messages contain job ids, never file bytes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
|
|
import aio_pika
|
|
|
|
from app import config, telemetry
|
|
|
|
logger = logging.getLogger("messenger.broker")
|
|
|
|
EXCHANGE = "attachment.jobs"
|
|
QUEUE = "attachment.jobs"
|
|
RETRY_QUEUE = "attachment.jobs.retry"
|
|
DLQ = "attachment.jobs.dlq"
|
|
ROUTING_KEY = "process"
|
|
|
|
_lock = asyncio.Lock()
|
|
_connection: aio_pika.RobustConnection | None = None
|
|
_channel: aio_pika.RobustChannel | None = None
|
|
_exchange: aio_pika.Exchange | None = None
|
|
|
|
|
|
async def _ensure() -> tuple[aio_pika.RobustChannel, aio_pika.Exchange]:
|
|
global _connection, _channel, _exchange
|
|
async with _lock:
|
|
if _channel is not None and not _channel.is_closed:
|
|
assert _exchange is not None
|
|
return _channel, _exchange
|
|
if not config.BROKER_URL:
|
|
raise RuntimeError("BROKER_URL is not set")
|
|
_connection = await aio_pika.connect_robust(config.BROKER_URL)
|
|
_channel = await _connection.channel(publisher_confirms=True)
|
|
_exchange = await _channel.declare_exchange(EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True)
|
|
await _channel.declare_queue(
|
|
RETRY_QUEUE,
|
|
durable=True,
|
|
arguments={
|
|
"x-dead-letter-exchange": EXCHANGE,
|
|
"x-dead-letter-routing-key": ROUTING_KEY,
|
|
},
|
|
)
|
|
await _channel.declare_queue(DLQ, durable=True)
|
|
queue = await _channel.declare_queue(QUEUE, durable=True)
|
|
await queue.bind(_exchange, routing_key=ROUTING_KEY)
|
|
return _channel, _exchange
|
|
|
|
|
|
async def publish_job(payload: dict, traceparent: str | None = None, tracestate: str | None = None) -> None:
|
|
channel, exchange = await _ensure()
|
|
parent = telemetry.remote_context(traceparent, tracestate)
|
|
with telemetry.span(
|
|
"attachment.job.publish",
|
|
context=parent,
|
|
kind="producer",
|
|
attributes={
|
|
"messaging.system": "rabbitmq",
|
|
"messaging.destination.name": QUEUE,
|
|
"messaging.operation": "publish",
|
|
"job.id": payload.get("job_id"),
|
|
},
|
|
):
|
|
headers = telemetry.current_carrier() or None
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
await exchange.publish(
|
|
aio_pika.Message(
|
|
body=body,
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
message_id=payload["job_id"],
|
|
headers=headers,
|
|
),
|
|
routing_key=ROUTING_KEY,
|
|
mandatory=True,
|
|
)
|
|
|
|
|
|
async def publish_delay(payload: dict, delay_seconds: float) -> None:
|
|
channel, _exchange = await _ensure()
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
milliseconds = max(1, int(delay_seconds * 1000))
|
|
await channel.default_exchange.publish(
|
|
aio_pika.Message(
|
|
body=body,
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
expiration=str(milliseconds),
|
|
message_id=payload["job_id"],
|
|
),
|
|
routing_key=RETRY_QUEUE,
|
|
mandatory=True,
|
|
)
|
|
|
|
|
|
async def publish_dlq(payload: dict) -> None:
|
|
channel, _exchange = await _ensure()
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
await channel.default_exchange.publish(
|
|
aio_pika.Message(
|
|
body=body,
|
|
content_type="application/json",
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
message_id=payload["job_id"],
|
|
),
|
|
routing_key=DLQ,
|
|
mandatory=True,
|
|
)
|
|
|
|
|
|
async def consume(handler, prefetch: int):
|
|
channel, _exchange = await _ensure()
|
|
await channel.set_qos(prefetch_count=max(1, prefetch))
|
|
queue = await channel.declare_queue(QUEUE, durable=True)
|
|
tag = await queue.consume(handler)
|
|
return queue, tag
|
|
|
|
|
|
async def close() -> None:
|
|
global _connection, _channel, _exchange
|
|
async with _lock:
|
|
if _connection is not None:
|
|
await _connection.close()
|
|
_connection = None
|
|
_channel = None
|
|
_exchange = None
|
|
|
|
|
|
def payload_from(raw: bytes) -> dict:
|
|
parsed = json.loads(raw.decode("utf-8"))
|
|
required = ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at")
|
|
if not isinstance(parsed, dict) or any(key not in parsed for key in required):
|
|
raise ValueError("job payload is incomplete")
|
|
if parsed["schema_version"] != 1 or parsed["type"] != "attachment.process.v1":
|
|
raise ValueError("job payload has the wrong type")
|
|
return parsed
|