Files
mtusi2026lab7/dispatcher.py

266 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
Цикл:
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
"""
import logging
import signal
import threading
import time
from pathlib import Path
import psycopg
from opentelemetry.trace import SpanKind
from psycopg.rows import dict_row
import dbconn
import secretstore
import settings
import telemetry
telemetry.setup("messenger-dispatcher")
import broker # noqa: E402
import faults # noqa: E402
import pipeline # noqa: E402
log = logging.getLogger("messenger.dispatcher")
# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs,
# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому
# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик
# только читает последний снимок (не ходит в БД из потока экспорта).
STATS_SECONDS = 5
STATS = {"at": 0.0, "db": None, "broker": None}
PUBLISHED = telemetry.meter.create_counter(
"messenger.messages.published", unit="{message}",
description="Outbox rows published to the broker (outcome: ok/error)")
QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE}
def _fresh(kind):
return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None
def _observe(kind, build):
def callback(_options):
from opentelemetry.metrics import Observation
data = _fresh(kind)
# lab7: ресурс, невидимый identity (RabbitMQ 4.2+), даёт None — такую точку не пишем
return [Observation(value, attrs) for value, attrs in build(data) if value is not None] if data else []
return callback
telemetry.meter.create_observable_gauge(
"messenger.queue.depth", [_observe("broker", lambda d: [
(d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])],
unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)")
telemetry.meter.create_observable_gauge(
"messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])],
unit="{consumer}", description="Consumers subscribed to the queue")
telemetry.meter.create_observable_gauge(
"messenger.jobs.backlog", [_observe("db", lambda d: [
(d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}),
(d["processing"], {"status": "processing"})])],
unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing")
telemetry.meter.create_observable_gauge(
"messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])],
unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits")
telemetry.meter.create_observable_gauge(
"messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])],
unit="{message}", description="Outbox rows due but not yet published")
telemetry.meter.create_observable_gauge(
"messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])],
unit="s", description="Age of the oldest due unpublished outbox row (publish lag)")
def collect_stats(conn, publisher) -> None:
"""Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот."""
if time.monotonic() - STATS["at"] < STATS_SECONDS:
return
db_stats = conn.execute(
"""SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due,
count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed,
count(*) FILTER (WHERE status = 'processing') AS processing,
coalesce(extract(epoch FROM now() - min(available_at)
FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age
FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone()
outbox = conn.execute(
"""SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due,
coalesce(extract(epoch FROM now() - min(available_at)
FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age
FROM outbox WHERE published_at IS NULL""").fetchone()
STATS["db"] = {**db_stats, **outbox}
try:
STATS["broker"] = broker.check_topology(publisher.open())
except broker.BrokerError:
STATS["broker"] = None
publisher.close()
STATS["at"] = time.monotonic()
STOP = threading.Event()
ACTOR = "dispatcher@" + settings.INSTANCE_ID
def connect_db():
# lab7: TLS verify-full, своя роль PostgreSQL, пароль из OpenBao (dbconn)
return dbconn.connect(autocommit=True, row_factory=dict_row,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def publish_due(conn, publisher: broker.Publisher) -> int:
published = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, job_id, attempt, payload FROM outbox
WHERE published_at IS NULL AND available_at <= now()
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
(settings.DISPATCHER_BATCH,),
).fetchall()
if not rows:
return published
for row in rows:
publish_one(conn, publisher, row)
published += 1
log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
if len(rows) < settings.DISPATCHER_BATCH:
return published
return published
def publish_one(conn, publisher: broker.Publisher, row: dict) -> None:
"""Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст
уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу."""
message = dict(row["payload"])
job_id, attempt = message.get("job_id"), message.get("attempt")
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send",
"messaging.destination.name": pipeline.EXCHANGE,
"messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY,
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
"messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]}
parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate"))
with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent,
kind=SpanKind.PRODUCER, attributes=attributes,
record_exception=False, set_status_on_exception=False) as span:
carrier = telemetry.inject()
if carrier.get("traceparent"):
message["traceparent"] = carrier["traceparent"]
message.pop("tracestate", None)
if carrier.get("tracestate"):
message["tracestate"] = carrier["tracestate"]
faults.crash("dispatcher_crash_before_publish")
try:
publisher.publish(message)
except broker.BrokerError as exc:
telemetry.mark_error(span, type(exc).__name__)
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"})
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__,
extra={"event": "outbox.publish_failed", "job.id": str(job_id),
"error.type": type(exc).__name__})
publisher.close()
raise
faults.crash("dispatcher_crash_after_confirm")
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"})
def note_failure(conn) -> None:
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
try:
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
WHERE published_at IS NULL AND available_at <= now()""")
except psycopg.Error:
pass
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def run() -> None:
publisher = broker.Publisher()
conn = None
last_leases = last_reconcile = 0.0
pause = 1.0
while not STOP.is_set():
try:
if conn is None or conn.closed:
conn = connect_db()
conn.execute("LISTEN outbox")
log.info("connected to PostgreSQL, listening for outbox notifications",
extra={"event": "db.connected"})
touch_alive()
publish_due(conn, publisher)
collect_stats(conn, publisher)
now = time.monotonic()
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
last_leases = now
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
log.warning("recovered %s job(s) with expired lease", recovered,
extra={"event": "lease.recovered", "count": recovered})
if now - last_reconcile >= 30:
last_reconcile = now
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
log.warning("re-published %s queued job(s) without a message in flight", lost,
extra={"event": "outbox.republished", "count": lost})
pause = 1.0
publisher.pump()
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
pass
except broker.BrokerError:
if conn is not None and not conn.closed:
note_failure(conn)
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause,
extra={"event": "broker.unavailable"})
STOP.wait(pause)
pause = min(pause * 2, 15)
except psycopg.Error as exc:
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause,
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
try:
if conn is not None:
conn.close()
except psycopg.Error:
pass
conn = None
STOP.wait(pause)
pause = min(pause * 2, 15)
publisher.close()
if conn is not None:
conn.close()
log.info("stopped", extra={"event": "service.stopped"})
telemetry.shutdown()
def main() -> None:
if not settings.BROKER_URL:
raise SystemExit("BROKER_URL is required")
secretstore.install_metrics(telemetry.meter)
secretstore.store.start() # lab7: пароль БД dispatcher — из OpenBao (CN=dispatcher)
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
run()
if __name__ == "__main__":
main()