forked from mtusi2026/mtusi2026lab6
Перенесена реализация lab5 (API x2 за HAProxy, PostgreSQL, SeaweedFS S3, RabbitMQ, relay outbox, worker x2, janitor) без изменения бизнес-API. Наблюдаемость: - OpenTelemetry SDK в API, relay, worker, janitor: traces, metrics, logs по OTLP/HTTP; структурные JSON-логи с trace_id/span_id - OTel Collector (memory_limiter, batch, ограниченные очереди и retry), Prometheus (метрики, alert rules), Loki (логи), Tempo (трассы), Grafana с provisioning datasources и dashboard - W3C trace context API -> outbox -> RabbitMQ -> worker (parent-child, включая retry); метрики очереди, пула БД, загрузки worker, CPU/RAM - SLI/SLO, 10 alert rules с unit-тестами promtool и runbooks - инциденты: остановка worker, блокировка БД, отказ telemetry backend; по наблюдениям исправлен простой worker в цикле AMQP (разбор x2.4)
348 lines
18 KiB
Python
348 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""Outbox relay + reaper (compose-сервис `relay`).
|
||
|
||
Закрывает разрыв PostgreSQL <-> broker (transactional outbox):
|
||
API в ОДНОЙ транзакции пишет queued-вложение, jobs и outbox. Relay берёт неопубликованные
|
||
строки (FOR UPDATE SKIP LOCKED — несколько relay не мешают друг другу), публикует каждую
|
||
в RabbitMQ с publisher confirm (persistent, mandatory) и только ПОСЛЕ подтверждения
|
||
брокера ставит published_at в той же транзакции. Крах между COMMIT API и publish, между
|
||
publish и confirm или между confirm и UPDATE outbox -> строка остаётся неопубликованной и
|
||
будет опубликована снова: возможен дубль сообщения (at-least-once), его гасит worker по
|
||
состоянию jobs.
|
||
|
||
Будит relay NOTIFY из триггера outbox; без уведомления — опрос раз в RELAY_POLL_SECONDS.
|
||
Строки с available_at в будущем (retry с backoff) публикуются, когда наступит срок.
|
||
|
||
Reaper (раз в REAPER_INTERVAL_SECONDS, под advisory-блокировкой — один на всех):
|
||
* processing с истёкшей арендой -> queued (backoff) или failed(retry_exhausted);
|
||
* queued без движения дольше JOB_REDRIVE_AFTER_SECONDS и без неопубликованного outbox
|
||
-> ещё одна публикация (сообщение могло потеряться вместе с volume брокера);
|
||
* опубликованные строки outbox старше OUTBOX_RETENTION_SECONDS удаляются.
|
||
|
||
Учебная инъекция отказа: RELAY_FAULT=crash-before-publish | crash-after-publish —
|
||
один раз за жизнь контейнера (маркер в /tmp) процесс делает os._exit посреди батча.
|
||
|
||
Lab6:
|
||
* каждое сообщение публикуется в PRODUCER span «attachments send» — child контекста из
|
||
payload.traceparent (span «create» API/worker). Контекст send записывается и в тело
|
||
(traceparent/tracestate, по jobs.schema.json), и в AMQP headers — worker продолжает трассу;
|
||
* метрики: messenger.outbox.published{outcome}, messenger.outbox.publish_delay (s, от
|
||
available_at до publisher confirm), и наблюдаемые gauges очереди по источнику истины
|
||
(PostgreSQL): messenger.jobs.count{status}, messenger.jobs.oldest_age{status} (s, возраст
|
||
старейшего незавершённого job от durable acceptance), messenger.jobs.ready_lag (s, сколько
|
||
ждёт самый старый job, которому уже пора выполняться), messenger.outbox.pending и
|
||
messenger.outbox.oldest_age (не опубликовано relay). Gauges считает отдельное соединение
|
||
со statement_timeout 2 с раз в интервал экспорта метрик.
|
||
"""
|
||
import json
|
||
import logging
|
||
import os
|
||
import sys
|
||
import threading
|
||
import time
|
||
|
||
import pika
|
||
import psycopg
|
||
from psycopg.rows import dict_row
|
||
|
||
import common
|
||
import jobs
|
||
import telemetry
|
||
from telemetry import SpanKind, Status, StatusCode
|
||
|
||
log = logging.getLogger("relay")
|
||
POLL_SECONDS = common.env_float("RELAY_POLL_SECONDS", 1)
|
||
BATCH = common.env_int("RELAY_BATCH", 100)
|
||
REAPER_INTERVAL = common.env_float("REAPER_INTERVAL_SECONDS", 5)
|
||
REDRIVE_AFTER = common.env_int("JOB_REDRIVE_AFTER_SECONDS", 900)
|
||
OUTBOX_RETENTION = common.env_int("OUTBOX_RETENTION_SECONDS", 86400)
|
||
RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2)
|
||
RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60)
|
||
FAULT = os.environ.get("RELAY_FAULT", "")
|
||
FAULT_MARKER = "/tmp/relay-fault.done"
|
||
HEARTBEAT = "/tmp/relay.heartbeat"
|
||
REAPER_LOCK_ID = 727003
|
||
|
||
PUBLISHED = telemetry.meter.create_counter(
|
||
"messenger.outbox.published", unit="{message}", description="Outbox rows published to the broker")
|
||
PUBLISH_DELAY = telemetry.meter.create_histogram(
|
||
"messenger.outbox.publish_delay", unit="s",
|
||
description="Delay from outbox available_at to broker publisher confirm")
|
||
|
||
|
||
class QueueGauges:
|
||
"""Глубина очереди и возраст заданий по PostgreSQL (источник истины), не по брокеру."""
|
||
QUERY = """
|
||
SELECT
|
||
(SELECT json_object_agg(status, n) FROM (SELECT status, count(*) AS n FROM jobs GROUP BY status) c) AS counts,
|
||
(SELECT json_object_agg(status, age) FROM (
|
||
SELECT status, EXTRACT(EPOCH FROM now() - min(created_at))::float8 AS age FROM jobs
|
||
WHERE status IN ('queued', 'processing') GROUP BY status) a) AS ages,
|
||
(SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(next_attempt_at)), 0)::float8 FROM jobs
|
||
WHERE status = 'queued' AND next_attempt_at <= now()) AS ready_lag,
|
||
(SELECT count(*) FROM outbox WHERE published_at IS NULL AND available_at <= now()) AS outbox_pending,
|
||
(SELECT COALESCE(EXTRACT(EPOCH FROM now() - min(available_at)), 0)::float8 FROM outbox
|
||
WHERE published_at IS NULL AND available_at <= now()) AS outbox_age"""
|
||
STATUSES = ("queued", "processing", "ready", "failed")
|
||
|
||
def __init__(self, conninfo):
|
||
self.conninfo = conninfo
|
||
self.conn = None
|
||
self.lock = threading.Lock()
|
||
self.cache = (0.0, None)
|
||
|
||
def snapshot(self):
|
||
# Несколько callbacks за один сбор метрик -> один SQL-запрос (кэш на 2 с).
|
||
with self.lock:
|
||
at, data = self.cache
|
||
if data is not None and time.monotonic() - at < 2:
|
||
return data
|
||
try:
|
||
if self.conn is None or self.conn.closed:
|
||
self.conn = psycopg.connect(self.conninfo, autocommit=True, row_factory=dict_row,
|
||
options="-c statement_timeout=2000")
|
||
data = self.conn.execute(self.QUERY).fetchone()
|
||
except psycopg.Error:
|
||
try:
|
||
if self.conn is not None:
|
||
self.conn.close()
|
||
except Exception:
|
||
pass
|
||
self.conn, data = None, None
|
||
self.cache = (time.monotonic(), data)
|
||
return data
|
||
|
||
def register(self):
|
||
def counts(options):
|
||
d = self.snapshot()
|
||
if d is not None:
|
||
got = d["counts"] or {}
|
||
for st in self.STATUSES:
|
||
yield telemetry.metrics.Observation(int(got.get(st, 0)), {"messenger.job.status": st})
|
||
|
||
def ages(options):
|
||
d = self.snapshot()
|
||
if d is not None:
|
||
got = d["ages"] or {}
|
||
for st in ("queued", "processing"):
|
||
yield telemetry.metrics.Observation(float(got.get(st) or 0.0), {"messenger.job.status": st})
|
||
|
||
def ready_lag(options):
|
||
d = self.snapshot()
|
||
if d is not None:
|
||
yield telemetry.metrics.Observation(float(d["ready_lag"] or 0.0))
|
||
|
||
def outbox_pending(options):
|
||
d = self.snapshot()
|
||
if d is not None:
|
||
yield telemetry.metrics.Observation(int(d["outbox_pending"] or 0))
|
||
|
||
def outbox_age(options):
|
||
d = self.snapshot()
|
||
if d is not None:
|
||
yield telemetry.metrics.Observation(float(d["outbox_age"] or 0.0))
|
||
|
||
m = telemetry.meter
|
||
m.create_observable_gauge("messenger.jobs.count", [counts], unit="{job}",
|
||
description="Jobs by status (PostgreSQL jobs table)")
|
||
m.create_observable_gauge("messenger.jobs.oldest_age", [ages], unit="s",
|
||
description="Age of the oldest unfinished job since durable acceptance")
|
||
m.create_observable_gauge("messenger.jobs.ready_lag", [ready_lag], unit="s",
|
||
description="How long the oldest due queued job has been waiting for a worker")
|
||
m.create_observable_gauge("messenger.outbox.pending", [outbox_pending], unit="{message}",
|
||
description="Due outbox rows not yet published to the broker")
|
||
m.create_observable_gauge("messenger.outbox.oldest_age", [outbox_age], unit="s",
|
||
description="Age of the oldest due unpublished outbox row")
|
||
|
||
|
||
def fault(point):
|
||
if FAULT != point or os.path.exists(FAULT_MARKER):
|
||
return
|
||
with open(FAULT_MARKER, "w") as f:
|
||
f.write(point)
|
||
log.warning("FAULT INJECTION %s: exiting without finishing the batch", point,
|
||
extra={"event": "fault.injected", "fault.point": point})
|
||
os._exit(86)
|
||
|
||
|
||
class Relay:
|
||
def __init__(self):
|
||
self.conninfo = common.database_conninfo()
|
||
self.db = None
|
||
self.listen = None
|
||
self.broker = None
|
||
self.channel = None
|
||
self.next_reap = 0.0
|
||
self.stop = threading.Event()
|
||
|
||
# --- соединения ---
|
||
def ensure_db(self):
|
||
if self.db is None or self.db.closed:
|
||
self.db = psycopg.connect(self.conninfo, row_factory=dict_row)
|
||
if self.listen is None or self.listen.closed:
|
||
self.listen = psycopg.connect(self.conninfo, autocommit=True)
|
||
self.listen.execute("LISTEN outbox")
|
||
|
||
def ensure_broker(self):
|
||
if self.broker is None or self.broker.is_closed or self.channel is None or self.channel.is_closed:
|
||
self.close_broker()
|
||
self.broker = common.connect_broker(log, self.stop)
|
||
if self.broker is None:
|
||
return
|
||
self.channel = self.broker.channel()
|
||
# Publisher confirms: basic_publish ждёт ack брокера (сообщение записано в
|
||
# quorum-очередь) и бросает NackError/UnroutableError при отказе.
|
||
self.channel.confirm_delivery()
|
||
log.info("connected to broker", extra={"event": "broker.connected"})
|
||
|
||
def close_broker(self):
|
||
try:
|
||
if self.broker is not None and not self.broker.is_closed:
|
||
self.broker.close()
|
||
except Exception:
|
||
pass
|
||
self.broker = self.channel = None
|
||
|
||
def close_db(self):
|
||
for conn in (self.db, self.listen):
|
||
try:
|
||
if conn is not None:
|
||
conn.close()
|
||
except Exception:
|
||
pass
|
||
self.db = self.listen = None
|
||
|
||
# --- публикация ---
|
||
def publish_batch(self):
|
||
"""Один батч в одной транзакции. Возвращает число опубликованных строк."""
|
||
spans = []
|
||
try:
|
||
with self.db.transaction():
|
||
rows = self.db.execute(
|
||
"""SELECT id, job_id, payload, EXTRACT(EPOCH FROM now() - available_at)::float8 AS delay_s
|
||
FROM outbox WHERE published_at IS NULL AND available_at <= now()
|
||
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", (BATCH,)).fetchall()
|
||
if not rows:
|
||
return 0
|
||
fault("crash-before-publish") # COMMIT задания есть, publish ещё нет
|
||
for row in rows:
|
||
payload = dict(row["payload"])
|
||
parent = telemetry.extract({k: payload[k] for k in ("traceparent", "tracestate") if k in payload})
|
||
span = telemetry.tracer.start_span(
|
||
"attachments send", kind=SpanKind.PRODUCER, context=parent,
|
||
attributes={"messaging.system": "rabbitmq", "messaging.operation.type": "send",
|
||
"messaging.operation.name": "publish",
|
||
"messaging.destination.name": common.EXCHANGE,
|
||
"messaging.rabbitmq.destination.routing_key": common.ROUTING_KEY,
|
||
"messaging.message.id": "%s:%s" % (payload["job_id"], payload["attempt"]),
|
||
"messenger.job.id": payload["job_id"],
|
||
"messenger.attachment.id": payload["attachment_id"],
|
||
"messenger.job.attempt": payload["attempt"],
|
||
"messenger.outbox.id": row["id"],
|
||
"messenger.outbox.publish_delay_s": round(row["delay_s"] or 0.0, 3)})
|
||
spans.append((span, row["delay_s"] or 0.0))
|
||
with telemetry.trace.use_span(span, end_on_exit=False):
|
||
carrier = telemetry.inject_dict()
|
||
payload.pop("tracestate", None)
|
||
payload.update(carrier)
|
||
self.channel.basic_publish(
|
||
exchange=common.EXCHANGE, routing_key=common.ROUTING_KEY,
|
||
body=json.dumps(payload, separators=(",", ":")).encode(),
|
||
properties=pika.BasicProperties(
|
||
content_type="application/json", delivery_mode=pika.DeliveryMode.Persistent,
|
||
message_id="%s:%s" % (payload["job_id"], payload["attempt"]), type=jobs.JOB_TYPE,
|
||
headers=dict({"x-outbox-id": row["id"]}, **carrier)),
|
||
mandatory=True)
|
||
fault("crash-after-publish") # confirm получен, published_at не записан -> дубль
|
||
self.db.execute(
|
||
"UPDATE outbox SET published_at = now(), publish_count = publish_count + 1 WHERE id = ANY(%s)",
|
||
([r["id"] for r in rows],))
|
||
except BaseException as exc:
|
||
# Транзакция откатилась: строки останутся неопубликованными и уйдут следующим батчем.
|
||
for span, _delay in spans:
|
||
span.set_attribute("error.type", type(exc).__name__)
|
||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||
span.end()
|
||
if spans:
|
||
PUBLISHED.add(len(spans), {"outcome": "failed", "error.type": type(exc).__name__})
|
||
raise
|
||
for span, delay in spans:
|
||
PUBLISH_DELAY.record(delay)
|
||
span.end()
|
||
PUBLISHED.add(len(rows), {"outcome": "ok"})
|
||
log.info("published %d job message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
|
||
return len(rows)
|
||
|
||
def reap(self):
|
||
with self.db.transaction():
|
||
if not self.db.execute("SELECT pg_try_advisory_xact_lock(%s) AS ok", (REAPER_LOCK_ID,)).fetchone()["ok"]:
|
||
return
|
||
expired = jobs.reap_expired_leases(self.db, RETRY_BASE, RETRY_MAX, actor="reaper:" + common.OWNER)
|
||
redriven = jobs.redrive_stale_queued(self.db, REDRIVE_AFTER, actor="reaper:" + common.OWNER)
|
||
purged = self.db.execute(
|
||
"DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)",
|
||
(OUTBOX_RETENTION,)).rowcount
|
||
if expired or redriven or purged:
|
||
log.info("reaper: expired_leases=%d redriven=%d outbox_purged=%d", expired, redriven, purged,
|
||
extra={"event": "reaper.pass", "expired_leases": expired, "redriven": redriven,
|
||
"outbox_purged": purged})
|
||
|
||
def wait_notify(self, timeout):
|
||
"""Ждём NOTIFY outbox не дольше timeout; параллельно обслуживаем heartbeat брокера."""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
if self.broker is not None:
|
||
self.broker.process_data_events(time_limit=0)
|
||
for _ in self.listen.notifies(timeout=min(0.25, max(0.01, deadline - time.monotonic())), stop_after=1):
|
||
return
|
||
|
||
def run(self):
|
||
log.info("relay started owner=%s fault=%s", common.OWNER, FAULT or "-", extra={"event": "relay.started"})
|
||
delay = 0.5
|
||
while not self.stop.is_set():
|
||
common.touch(HEARTBEAT)
|
||
try:
|
||
self.ensure_db()
|
||
self.ensure_broker()
|
||
if self.broker is None:
|
||
continue
|
||
while self.publish_batch() == BATCH:
|
||
pass
|
||
if time.monotonic() >= self.next_reap:
|
||
self.reap()
|
||
self.next_reap = time.monotonic() + REAPER_INTERVAL
|
||
self.wait_notify(POLL_SECONDS)
|
||
delay = 0.5
|
||
except (pika.exceptions.AMQPError, pika.exceptions.UnroutableError, pika.exceptions.NackError) as exc:
|
||
# Транзакция батча откатилась: строки остались неопубликованными.
|
||
log.warning("broker publish failed (%s), retry in %.1fs", type(exc).__name__, delay,
|
||
extra={"event": "broker.publish_failed", "error.type": type(exc).__name__})
|
||
self.close_broker()
|
||
self.stop.wait(delay)
|
||
delay = min(5.0, delay * 2)
|
||
except psycopg.Error as exc:
|
||
log.warning("database error (%s), retry in %.1fs", type(exc).__name__, delay,
|
||
extra={"event": "db.error", "error.type": type(exc).__name__})
|
||
self.close_db()
|
||
self.stop.wait(delay)
|
||
delay = min(5.0, delay * 2)
|
||
self.close_broker()
|
||
self.close_db()
|
||
|
||
|
||
def main():
|
||
import signal
|
||
telemetry.setup("messenger-relay")
|
||
relay = Relay()
|
||
QueueGauges(relay.conninfo).register()
|
||
signal.signal(signal.SIGTERM, lambda *_: relay.stop.set())
|
||
signal.signal(signal.SIGINT, lambda *_: relay.stop.set())
|
||
relay.run()
|
||
log.info("relay stopped", extra={"event": "relay.stopped"})
|
||
telemetry.shutdown()
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|