Files
mtusi2026lab5/dispatcher.py
T

151 lines
6.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
Цикл:
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
"""
import logging
import signal
import threading
import time
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
import broker
import faults
import pipeline
import settings
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
log = logging.getLogger("messenger.dispatcher")
STOP = threading.Event()
ACTOR = "dispatcher@" + settings.INSTANCE_ID
def connect_db():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def publish_due(conn, publisher: broker.Publisher) -> int:
published = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, job_id, attempt, payload FROM outbox
WHERE published_at IS NULL AND available_at <= now()
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
(settings.DISPATCHER_BATCH,),
).fetchall()
if not rows:
return published
for row in rows:
faults.crash("dispatcher_crash_before_publish")
try:
publisher.publish(row["payload"])
except broker.BrokerError as exc:
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__)
publisher.close()
raise
faults.crash("dispatcher_crash_after_confirm")
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
published += 1
log.info("published %s message(s)", len(rows))
if len(rows) < settings.DISPATCHER_BATCH:
return published
return published
def note_failure(conn) -> None:
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
try:
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
WHERE published_at IS NULL AND available_at <= now()""")
except psycopg.Error:
pass
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def run() -> None:
publisher = broker.Publisher()
conn = None
last_leases = last_reconcile = 0.0
pause = 1.0
while not STOP.is_set():
try:
if conn is None or conn.closed:
conn = connect_db()
conn.execute("LISTEN outbox")
log.info("connected to PostgreSQL, listening for outbox notifications")
touch_alive()
publish_due(conn, publisher)
now = time.monotonic()
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
last_leases = now
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
log.warning("recovered %s job(s) with expired lease", recovered)
if now - last_reconcile >= 30:
last_reconcile = now
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
log.warning("re-published %s queued job(s) without a message in flight", lost)
pause = 1.0
publisher.pump()
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
pass
except broker.BrokerError:
if conn is not None and not conn.closed:
note_failure(conn)
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause)
STOP.wait(pause)
pause = min(pause * 2, 15)
except psycopg.Error as exc:
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause)
try:
if conn is not None:
conn.close()
except psycopg.Error:
pass
conn = None
STOP.wait(pause)
pause = min(pause * 2, 15)
publisher.close()
if conn is not None:
conn.close()
log.info("stopped")
def main() -> None:
if not settings.DATABASE_URL or not settings.BROKER_URL:
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
"ON" if settings.FAULT_INJECTION else "off")
run()
if __name__ == "__main__":
main()