forked from mtusi2026/mtusi2026lab5
151 lines
6.5 KiB
Python
151 lines
6.5 KiB
Python
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
|
||
|
||
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
|
||
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
|
||
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
|
||
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
|
||
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
|
||
|
||
Цикл:
|
||
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
|
||
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
|
||
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
|
||
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
|
||
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
|
||
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
|
||
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
|
||
"""
|
||
import logging
|
||
import signal
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import psycopg
|
||
from psycopg.rows import dict_row
|
||
|
||
import broker
|
||
import faults
|
||
import pipeline
|
||
import settings
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||
log = logging.getLogger("messenger.dispatcher")
|
||
|
||
STOP = threading.Event()
|
||
ACTOR = "dispatcher@" + settings.INSTANCE_ID
|
||
|
||
|
||
def connect_db():
|
||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||
|
||
|
||
def publish_due(conn, publisher: broker.Publisher) -> int:
|
||
published = 0
|
||
while not STOP.is_set():
|
||
with conn.transaction():
|
||
rows = conn.execute(
|
||
"""SELECT id, job_id, attempt, payload FROM outbox
|
||
WHERE published_at IS NULL AND available_at <= now()
|
||
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||
(settings.DISPATCHER_BATCH,),
|
||
).fetchall()
|
||
if not rows:
|
||
return published
|
||
for row in rows:
|
||
faults.crash("dispatcher_crash_before_publish")
|
||
try:
|
||
publisher.publish(row["payload"])
|
||
except broker.BrokerError as exc:
|
||
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__)
|
||
publisher.close()
|
||
raise
|
||
faults.crash("dispatcher_crash_after_confirm")
|
||
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
|
||
published += 1
|
||
log.info("published %s message(s)", len(rows))
|
||
if len(rows) < settings.DISPATCHER_BATCH:
|
||
return published
|
||
return published
|
||
|
||
|
||
def note_failure(conn) -> None:
|
||
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
|
||
try:
|
||
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
|
||
WHERE published_at IS NULL AND available_at <= now()""")
|
||
except psycopg.Error:
|
||
pass
|
||
|
||
|
||
def touch_alive() -> None:
|
||
try:
|
||
Path(settings.ALIVE_FILE).touch()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def run() -> None:
|
||
publisher = broker.Publisher()
|
||
conn = None
|
||
last_leases = last_reconcile = 0.0
|
||
pause = 1.0
|
||
while not STOP.is_set():
|
||
try:
|
||
if conn is None or conn.closed:
|
||
conn = connect_db()
|
||
conn.execute("LISTEN outbox")
|
||
log.info("connected to PostgreSQL, listening for outbox notifications")
|
||
touch_alive()
|
||
publish_due(conn, publisher)
|
||
now = time.monotonic()
|
||
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
|
||
last_leases = now
|
||
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
|
||
log.warning("recovered %s job(s) with expired lease", recovered)
|
||
if now - last_reconcile >= 30:
|
||
last_reconcile = now
|
||
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
|
||
log.warning("re-published %s queued job(s) without a message in flight", lost)
|
||
pause = 1.0
|
||
publisher.pump()
|
||
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
|
||
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
|
||
pass
|
||
except broker.BrokerError:
|
||
if conn is not None and not conn.closed:
|
||
note_failure(conn)
|
||
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause)
|
||
STOP.wait(pause)
|
||
pause = min(pause * 2, 15)
|
||
except psycopg.Error as exc:
|
||
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause)
|
||
try:
|
||
if conn is not None:
|
||
conn.close()
|
||
except psycopg.Error:
|
||
pass
|
||
conn = None
|
||
STOP.wait(pause)
|
||
pause = min(pause * 2, 15)
|
||
publisher.close()
|
||
if conn is not None:
|
||
conn.close()
|
||
log.info("stopped")
|
||
|
||
|
||
def main() -> None:
|
||
if not settings.DATABASE_URL or not settings.BROKER_URL:
|
||
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
signal.signal(sig, lambda *_: STOP.set())
|
||
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
|
||
"ON" if settings.FAULT_INJECTION else "off")
|
||
run()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|