Files
mtusi2026lab6/src/relay.py
T
zemiles c199f6632a ЛР 6 — БПИ2303 — Лазарев — на 4
Перенесена реализация lab5 (API x2 за HAProxy, PostgreSQL, SeaweedFS S3,
RabbitMQ, relay outbox, worker x2, janitor) без изменения бизнес-API.

Наблюдаемость:
- OpenTelemetry SDK в API, relay, worker, janitor: traces, metrics, logs
  по OTLP/HTTP; структурные JSON-логи с trace_id/span_id
- OTel Collector (memory_limiter, batch, ограниченные очереди и retry),
  Prometheus (метрики, alert rules), Loki (логи), Tempo (трассы),
  Grafana с provisioning datasources и dashboard
- W3C trace context API -> outbox -> RabbitMQ -> worker (parent-child,
  включая retry); метрики очереди, пула БД, загрузки worker, CPU/RAM
- SLI/SLO, 10 alert rules с unit-тестами promtool и runbooks
- инциденты: остановка worker, блокировка БД, отказ telemetry backend;
  по наблюдениям исправлен простой worker в цикле AMQP (разбор x2.4)
2026-11-17 10:59:41 +03:00

348 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Outbox relay + reaper (compose-сервис `relay`).
Закрывает разрыв PostgreSQL <-> broker (transactional outbox):
API в ОДНОЙ транзакции пишет queued-вложение, jobs и outbox. Relay берёт неопубликованные
строки (FOR UPDATE SKIP LOCKED — несколько relay не мешают друг другу), публикует каждую
в RabbitMQ с publisher confirm (persistent, mandatory) и только ПОСЛЕ подтверждения
брокера ставит published_at в той же транзакции. Крах между COMMIT API и publish, между
publish и confirm или между confirm и UPDATE outbox -> строка остаётся неопубликованной и
будет опубликована снова: возможен дубль сообщения (at-least-once), его гасит worker по
состоянию jobs.
Будит relay NOTIFY из триггера outbox; без уведомления — опрос раз в RELAY_POLL_SECONDS.
Строки с available_at в будущем (retry с backoff) публикуются, когда наступит срок.
Reaper (раз в REAPER_INTERVAL_SECONDS, под advisory-блокировкой — один на всех):
* processing с истёкшей арендой -> queued (backoff) или failed(retry_exhausted);
* queued без движения дольше JOB_REDRIVE_AFTER_SECONDS и без неопубликованного outbox
-> ещё одна публикация (сообщение могло потеряться вместе с volume брокера);
* опубликованные строки outbox старше OUTBOX_RETENTION_SECONDS удаляются.
Учебная инъекция отказа: RELAY_FAULT=crash-before-publish | crash-after-publish —
один раз за жизнь контейнера (маркер в /tmp) процесс делает os._exit посреди батча.
Lab6:
* каждое сообщение публикуется в PRODUCER span «attachments send» — child контекста из
payload.traceparent (span «create» API/worker). Контекст send записывается и в тело
(traceparent/tracestate, по jobs.schema.json), и в AMQP headers — worker продолжает трассу;
* метрики: messenger.outbox.published{outcome}, messenger.outbox.publish_delay (s, от
available_at до publisher confirm), и наблюдаемые gauges очереди по источнику истины
(PostgreSQL): messenger.jobs.count{status}, messenger.jobs.oldest_age{status} (s, возраст
старейшего незавершённого job от durable acceptance), messenger.jobs.ready_lag (s, сколько
ждёт самый старый job, которому уже пора выполняться), messenger.outbox.pending и
messenger.outbox.oldest_age (не опубликовано relay). Gauges считает отдельное соединение
со statement_timeout 2 с раз в интервал экспорта метрик.
"""
import json
import logging
import os
import sys
import threading
import time
import pika
import psycopg
from psycopg.rows import dict_row
import common
import jobs
import telemetry
from telemetry import SpanKind, Status, StatusCode
log = logging.getLogger("relay")
POLL_SECONDS = common.env_float("RELAY_POLL_SECONDS", 1)
BATCH = common.env_int("RELAY_BATCH", 100)
REAPER_INTERVAL = common.env_float("REAPER_INTERVAL_SECONDS", 5)
REDRIVE_AFTER = common.env_int("JOB_REDRIVE_AFTER_SECONDS", 900)
OUTBOX_RETENTION = common.env_int("OUTBOX_RETENTION_SECONDS", 86400)
RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2)
RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60)
FAULT = os.environ.get("RELAY_FAULT", "")
FAULT_MARKER = "/tmp/relay-fault.done"
HEARTBEAT = "/tmp/relay.heartbeat"
REAPER_LOCK_ID = 727003
PUBLISHED = telemetry.meter.create_counter(
"messenger.outbox.published", unit="{message}", description="Outbox rows published to the broker")
PUBLISH_DELAY = telemetry.meter.create_histogram(
"messenger.outbox.publish_delay", unit="s",
description="Delay from outbox available_at to broker publisher confirm")
class QueueGauges:
"""Глубина очереди и возраст заданий по PostgreSQL (источник истины), не по брокеру."""
QUERY = """
SELECT
(SELECT json_object_agg(status, n) FROM (SELECT status, count(*) AS n FROM jobs GROUP BY status) c) AS counts,
(SELECT json_object_agg(status, age) FROM (
SELECT status, EXTRACT(EPOCH FROM now() - min(created_at))::float8 AS age FROM jobs
WHERE status IN ('queued', 'processing') GROUP BY status) a) AS ages,
(SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(next_attempt_at)), 0)::float8 FROM jobs
WHERE status = 'queued' AND next_attempt_at <= now()) AS ready_lag,
(SELECT count(*) FROM outbox WHERE published_at IS NULL AND available_at <= now()) AS outbox_pending,
(SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(available_at)), 0)::float8 FROM outbox
WHERE published_at IS NULL AND available_at <= now()) AS outbox_age"""
STATUSES = ("queued", "processing", "ready", "failed")
def __init__(self, conninfo):
self.conninfo = conninfo
self.conn = None
self.lock = threading.Lock()
self.cache = (0.0, None)
def snapshot(self):
# Несколько callbacks за один сбор метрик -> один SQL-запрос (кэш на 2 с).
with self.lock:
at, data = self.cache
if data is not None and time.monotonic() - at < 2:
return data
try:
if self.conn is None or self.conn.closed:
self.conn = psycopg.connect(self.conninfo, autocommit=True, row_factory=dict_row,
options="-c statement_timeout=2000")
data = self.conn.execute(self.QUERY).fetchone()
except psycopg.Error:
try:
if self.conn is not None:
self.conn.close()
except Exception:
pass
self.conn, data = None, None
self.cache = (time.monotonic(), data)
return data
def register(self):
def counts(options):
d = self.snapshot()
if d is not None:
got = d["counts"] or {}
for st in self.STATUSES:
yield telemetry.metrics.Observation(int(got.get(st, 0)), {"messenger.job.status": st})
def ages(options):
d = self.snapshot()
if d is not None:
got = d["ages"] or {}
for st in ("queued", "processing"):
yield telemetry.metrics.Observation(float(got.get(st) or 0.0), {"messenger.job.status": st})
def ready_lag(options):
d = self.snapshot()
if d is not None:
yield telemetry.metrics.Observation(float(d["ready_lag"] or 0.0))
def outbox_pending(options):
d = self.snapshot()
if d is not None:
yield telemetry.metrics.Observation(int(d["outbox_pending"] or 0))
def outbox_age(options):
d = self.snapshot()
if d is not None:
yield telemetry.metrics.Observation(float(d["outbox_age"] or 0.0))
m = telemetry.meter
m.create_observable_gauge("messenger.jobs.count", [counts], unit="{job}",
description="Jobs by status (PostgreSQL jobs table)")
m.create_observable_gauge("messenger.jobs.oldest_age", [ages], unit="s",
description="Age of the oldest unfinished job since durable acceptance")
m.create_observable_gauge("messenger.jobs.ready_lag", [ready_lag], unit="s",
description="How long the oldest due queued job has been waiting for a worker")
m.create_observable_gauge("messenger.outbox.pending", [outbox_pending], unit="{message}",
description="Due outbox rows not yet published to the broker")
m.create_observable_gauge("messenger.outbox.oldest_age", [outbox_age], unit="s",
description="Age of the oldest due unpublished outbox row")
def fault(point):
if FAULT != point or os.path.exists(FAULT_MARKER):
return
with open(FAULT_MARKER, "w") as f:
f.write(point)
log.warning("FAULT INJECTION %s: exiting without finishing the batch", point,
extra={"event": "fault.injected", "fault.point": point})
os._exit(86)
class Relay:
def __init__(self):
self.conninfo = common.database_conninfo()
self.db = None
self.listen = None
self.broker = None
self.channel = None
self.next_reap = 0.0
self.stop = threading.Event()
# --- соединения ---
def ensure_db(self):
if self.db is None or self.db.closed:
self.db = psycopg.connect(self.conninfo, row_factory=dict_row)
if self.listen is None or self.listen.closed:
self.listen = psycopg.connect(self.conninfo, autocommit=True)
self.listen.execute("LISTEN outbox")
def ensure_broker(self):
if self.broker is None or self.broker.is_closed or self.channel is None or self.channel.is_closed:
self.close_broker()
self.broker = common.connect_broker(log, self.stop)
if self.broker is None:
return
self.channel = self.broker.channel()
# Publisher confirms: basic_publish ждёт ack брокера (сообщение записано в
# quorum-очередь) и бросает NackError/UnroutableError при отказе.
self.channel.confirm_delivery()
log.info("connected to broker", extra={"event": "broker.connected"})
def close_broker(self):
try:
if self.broker is not None and not self.broker.is_closed:
self.broker.close()
except Exception:
pass
self.broker = self.channel = None
def close_db(self):
for conn in (self.db, self.listen):
try:
if conn is not None:
conn.close()
except Exception:
pass
self.db = self.listen = None
# --- публикация ---
def publish_batch(self):
"""Один батч в одной транзакции. Возвращает число опубликованных строк."""
spans = []
try:
with self.db.transaction():
rows = self.db.execute(
"""SELECT id, job_id, payload, EXTRACT(EPOCH FROM now() - available_at)::float8 AS delay_s
FROM outbox WHERE published_at IS NULL AND available_at <= now()
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", (BATCH,)).fetchall()
if not rows:
return 0
fault("crash-before-publish") # COMMIT задания есть, publish ещё нет
for row in rows:
payload = dict(row["payload"])
parent = telemetry.extract({k: payload[k] for k in ("traceparent", "tracestate") if k in payload})
span = telemetry.tracer.start_span(
"attachments send", kind=SpanKind.PRODUCER, context=parent,
attributes={"messaging.system": "rabbitmq", "messaging.operation.type": "send",
"messaging.operation.name": "publish",
"messaging.destination.name": common.EXCHANGE,
"messaging.rabbitmq.destination.routing_key": common.ROUTING_KEY,
"messaging.message.id": "%s:%s" % (payload["job_id"], payload["attempt"]),
"messenger.job.id": payload["job_id"],
"messenger.attachment.id": payload["attachment_id"],
"messenger.job.attempt": payload["attempt"],
"messenger.outbox.id": row["id"],
"messenger.outbox.publish_delay_s": round(row["delay_s"] or 0.0, 3)})
spans.append((span, row["delay_s"] or 0.0))
with telemetry.trace.use_span(span, end_on_exit=False):
carrier = telemetry.inject_dict()
payload.pop("tracestate", None)
payload.update(carrier)
self.channel.basic_publish(
exchange=common.EXCHANGE, routing_key=common.ROUTING_KEY,
body=json.dumps(payload, separators=(",", ":")).encode(),
properties=pika.BasicProperties(
content_type="application/json", delivery_mode=pika.DeliveryMode.Persistent,
message_id="%s:%s" % (payload["job_id"], payload["attempt"]), type=jobs.JOB_TYPE,
headers=dict({"x-outbox-id": row["id"]}, **carrier)),
mandatory=True)
fault("crash-after-publish") # confirm получен, published_at не записан -> дубль
self.db.execute(
"UPDATE outbox SET published_at = now(), publish_count = publish_count + 1 WHERE id = ANY(%s)",
([r["id"] for r in rows],))
except BaseException as exc:
# Транзакция откатилась: строки останутся неопубликованными и уйдут следующим батчем.
for span, _delay in spans:
span.set_attribute("error.type", type(exc).__name__)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
span.end()
if spans:
PUBLISHED.add(len(spans), {"outcome": "failed", "error.type": type(exc).__name__})
raise
for span, delay in spans:
PUBLISH_DELAY.record(delay)
span.end()
PUBLISHED.add(len(rows), {"outcome": "ok"})
log.info("published %d job message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
return len(rows)
def reap(self):
with self.db.transaction():
if not self.db.execute("SELECT pg_try_advisory_xact_lock(%s) AS ok", (REAPER_LOCK_ID,)).fetchone()["ok"]:
return
expired = jobs.reap_expired_leases(self.db, RETRY_BASE, RETRY_MAX, actor="reaper:" + common.OWNER)
redriven = jobs.redrive_stale_queued(self.db, REDRIVE_AFTER, actor="reaper:" + common.OWNER)
purged = self.db.execute(
"DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)",
(OUTBOX_RETENTION,)).rowcount
if expired or redriven or purged:
log.info("reaper: expired_leases=%d redriven=%d outbox_purged=%d", expired, redriven, purged,
extra={"event": "reaper.pass", "expired_leases": expired, "redriven": redriven,
"outbox_purged": purged})
def wait_notify(self, timeout):
"""Ждём NOTIFY outbox не дольше timeout; параллельно обслуживаем heartbeat брокера."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self.broker is not None:
self.broker.process_data_events(time_limit=0)
for _ in self.listen.notifies(timeout=min(0.25, max(0.01, deadline - time.monotonic())), stop_after=1):
return
def run(self):
log.info("relay started owner=%s fault=%s", common.OWNER, FAULT or "-", extra={"event": "relay.started"})
delay = 0.5
while not self.stop.is_set():
common.touch(HEARTBEAT)
try:
self.ensure_db()
self.ensure_broker()
if self.broker is None:
continue
while self.publish_batch() == BATCH:
pass
if time.monotonic() >= self.next_reap:
self.reap()
self.next_reap = time.monotonic() + REAPER_INTERVAL
self.wait_notify(POLL_SECONDS)
delay = 0.5
except (pika.exceptions.AMQPError, pika.exceptions.UnroutableError, pika.exceptions.NackError) as exc:
# Транзакция батча откатилась: строки остались неопубликованными.
log.warning("broker publish failed (%s), retry in %.1fs", type(exc).__name__, delay,
extra={"event": "broker.publish_failed", "error.type": type(exc).__name__})
self.close_broker()
self.stop.wait(delay)
delay = min(5.0, delay * 2)
except psycopg.Error as exc:
log.warning("database error (%s), retry in %.1fs", type(exc).__name__, delay,
extra={"event": "db.error", "error.type": type(exc).__name__})
self.close_db()
self.stop.wait(delay)
delay = min(5.0, delay * 2)
self.close_broker()
self.close_db()
def main():
import signal
telemetry.setup("messenger-relay")
relay = Relay()
QueueGauges(relay.conninfo).register()
signal.signal(signal.SIGTERM, lambda *_: relay.stop.set())
signal.signal(signal.SIGINT, lambda *_: relay.stop.set())
relay.run()
log.info("relay stopped", extra={"event": "relay.stopped"})
telemetry.shutdown()
return 0
if __name__ == "__main__":
sys.exit(main())