forked from mtusi2026/mtusi2026lab7
266 lines
13 KiB
Python
266 lines
13 KiB
Python
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
|
||
|
||
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
|
||
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
|
||
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
|
||
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
|
||
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
|
||
|
||
Цикл:
|
||
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
|
||
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
|
||
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
|
||
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
|
||
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
|
||
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
|
||
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
|
||
"""
|
||
import logging
|
||
import signal
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import psycopg
|
||
from opentelemetry.trace import SpanKind
|
||
from psycopg.rows import dict_row
|
||
|
||
import dbconn
|
||
import secretstore
|
||
import settings
|
||
import telemetry
|
||
|
||
telemetry.setup("messenger-dispatcher")
|
||
|
||
import broker # noqa: E402
|
||
import faults # noqa: E402
|
||
import pipeline # noqa: E402
|
||
|
||
log = logging.getLogger("messenger.dispatcher")
|
||
|
||
# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs,
|
||
# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому
|
||
# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик
|
||
# только читает последний снимок (не ходит в БД из потока экспорта).
|
||
STATS_SECONDS = 5
|
||
STATS = {"at": 0.0, "db": None, "broker": None}
|
||
PUBLISHED = telemetry.meter.create_counter(
|
||
"messenger.messages.published", unit="{message}",
|
||
description="Outbox rows published to the broker (outcome: ok/error)")
|
||
QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE}
|
||
|
||
|
||
def _fresh(kind):
|
||
return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None
|
||
|
||
|
||
def _observe(kind, build):
|
||
def callback(_options):
|
||
from opentelemetry.metrics import Observation
|
||
data = _fresh(kind)
|
||
# lab7: ресурс, невидимый identity (RabbitMQ 4.2+), даёт None — такую точку не пишем
|
||
return [Observation(value, attrs) for value, attrs in build(data) if value is not None] if data else []
|
||
return callback
|
||
|
||
|
||
telemetry.meter.create_observable_gauge(
|
||
"messenger.queue.depth", [_observe("broker", lambda d: [
|
||
(d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])],
|
||
unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)")
|
||
telemetry.meter.create_observable_gauge(
|
||
"messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])],
|
||
unit="{consumer}", description="Consumers subscribed to the queue")
|
||
telemetry.meter.create_observable_gauge(
|
||
"messenger.jobs.backlog", [_observe("db", lambda d: [
|
||
(d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}),
|
||
(d["processing"], {"status": "processing"})])],
|
||
unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing")
|
||
telemetry.meter.create_observable_gauge(
|
||
"messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])],
|
||
unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits")
|
||
telemetry.meter.create_observable_gauge(
|
||
"messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])],
|
||
unit="{message}", description="Outbox rows due but not yet published")
|
||
telemetry.meter.create_observable_gauge(
|
||
"messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])],
|
||
unit="s", description="Age of the oldest due unpublished outbox row (publish lag)")
|
||
|
||
|
||
def collect_stats(conn, publisher) -> None:
|
||
"""Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот."""
|
||
if time.monotonic() - STATS["at"] < STATS_SECONDS:
|
||
return
|
||
db_stats = conn.execute(
|
||
"""SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due,
|
||
count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed,
|
||
count(*) FILTER (WHERE status = 'processing') AS processing,
|
||
coalesce(extract(epoch FROM now() - min(available_at)
|
||
FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age
|
||
FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone()
|
||
outbox = conn.execute(
|
||
"""SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due,
|
||
coalesce(extract(epoch FROM now() - min(available_at)
|
||
FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age
|
||
FROM outbox WHERE published_at IS NULL""").fetchone()
|
||
STATS["db"] = {**db_stats, **outbox}
|
||
try:
|
||
STATS["broker"] = broker.check_topology(publisher.open())
|
||
except broker.BrokerError:
|
||
STATS["broker"] = None
|
||
publisher.close()
|
||
STATS["at"] = time.monotonic()
|
||
|
||
STOP = threading.Event()
|
||
ACTOR = "dispatcher@" + settings.INSTANCE_ID
|
||
|
||
|
||
def connect_db():
|
||
# lab7: TLS verify-full, своя роль PostgreSQL, пароль из OpenBao (dbconn)
|
||
return dbconn.connect(autocommit=True, row_factory=dict_row,
|
||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||
|
||
|
||
def publish_due(conn, publisher: broker.Publisher) -> int:
|
||
published = 0
|
||
while not STOP.is_set():
|
||
with conn.transaction():
|
||
rows = conn.execute(
|
||
"""SELECT id, job_id, attempt, payload FROM outbox
|
||
WHERE published_at IS NULL AND available_at <= now()
|
||
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||
(settings.DISPATCHER_BATCH,),
|
||
).fetchall()
|
||
if not rows:
|
||
return published
|
||
for row in rows:
|
||
publish_one(conn, publisher, row)
|
||
published += 1
|
||
log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
|
||
if len(rows) < settings.DISPATCHER_BATCH:
|
||
return published
|
||
return published
|
||
|
||
|
||
def publish_one(conn, publisher: broker.Publisher, row: dict) -> None:
|
||
"""Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст
|
||
уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу."""
|
||
message = dict(row["payload"])
|
||
job_id, attempt = message.get("job_id"), message.get("attempt")
|
||
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send",
|
||
"messaging.destination.name": pipeline.EXCHANGE,
|
||
"messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY,
|
||
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
|
||
"messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]}
|
||
parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate"))
|
||
with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent,
|
||
kind=SpanKind.PRODUCER, attributes=attributes,
|
||
record_exception=False, set_status_on_exception=False) as span:
|
||
carrier = telemetry.inject()
|
||
if carrier.get("traceparent"):
|
||
message["traceparent"] = carrier["traceparent"]
|
||
message.pop("tracestate", None)
|
||
if carrier.get("tracestate"):
|
||
message["tracestate"] = carrier["tracestate"]
|
||
faults.crash("dispatcher_crash_before_publish")
|
||
try:
|
||
publisher.publish(message)
|
||
except broker.BrokerError as exc:
|
||
telemetry.mark_error(span, type(exc).__name__)
|
||
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"})
|
||
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__,
|
||
extra={"event": "outbox.publish_failed", "job.id": str(job_id),
|
||
"error.type": type(exc).__name__})
|
||
publisher.close()
|
||
raise
|
||
faults.crash("dispatcher_crash_after_confirm")
|
||
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
|
||
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"})
|
||
|
||
|
||
def note_failure(conn) -> None:
|
||
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
|
||
try:
|
||
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
|
||
WHERE published_at IS NULL AND available_at <= now()""")
|
||
except psycopg.Error:
|
||
pass
|
||
|
||
|
||
def touch_alive() -> None:
|
||
try:
|
||
Path(settings.ALIVE_FILE).touch()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def run() -> None:
|
||
publisher = broker.Publisher()
|
||
conn = None
|
||
last_leases = last_reconcile = 0.0
|
||
pause = 1.0
|
||
while not STOP.is_set():
|
||
try:
|
||
if conn is None or conn.closed:
|
||
conn = connect_db()
|
||
conn.execute("LISTEN outbox")
|
||
log.info("connected to PostgreSQL, listening for outbox notifications",
|
||
extra={"event": "db.connected"})
|
||
touch_alive()
|
||
publish_due(conn, publisher)
|
||
collect_stats(conn, publisher)
|
||
now = time.monotonic()
|
||
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
|
||
last_leases = now
|
||
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
|
||
log.warning("recovered %s job(s) with expired lease", recovered,
|
||
extra={"event": "lease.recovered", "count": recovered})
|
||
if now - last_reconcile >= 30:
|
||
last_reconcile = now
|
||
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
|
||
log.warning("re-published %s queued job(s) without a message in flight", lost,
|
||
extra={"event": "outbox.republished", "count": lost})
|
||
pause = 1.0
|
||
publisher.pump()
|
||
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
|
||
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
|
||
pass
|
||
except broker.BrokerError:
|
||
if conn is not None and not conn.closed:
|
||
note_failure(conn)
|
||
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause,
|
||
extra={"event": "broker.unavailable"})
|
||
STOP.wait(pause)
|
||
pause = min(pause * 2, 15)
|
||
except psycopg.Error as exc:
|
||
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause,
|
||
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
|
||
try:
|
||
if conn is not None:
|
||
conn.close()
|
||
except psycopg.Error:
|
||
pass
|
||
conn = None
|
||
STOP.wait(pause)
|
||
pause = min(pause * 2, 15)
|
||
publisher.close()
|
||
if conn is not None:
|
||
conn.close()
|
||
log.info("stopped", extra={"event": "service.stopped"})
|
||
telemetry.shutdown()
|
||
|
||
|
||
def main() -> None:
|
||
if not settings.BROKER_URL:
|
||
raise SystemExit("BROKER_URL is required")
|
||
secretstore.install_metrics(telemetry.meter)
|
||
secretstore.store.start() # lab7: пароль БД dispatcher — из OpenBao (CN=dispatcher)
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
signal.signal(sig, lambda *_: STOP.set())
|
||
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
|
||
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
|
||
run()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|