From d753e348eb2dedca7acce5a0b70a5cc29adecc53 Mon Sep 17 00:00:00 2001 From: Dmitrii Grishko Date: Fri, 13 Nov 2026 18:52:25 +0300 Subject: [PATCH] feat: add pipeline tracing and observability alerts --- compose.yml | 3 +- infra/otel/alerts.yaml | 20 ++ infra/otel/grafana/dashboards/messenger.json | 308 ++++++++++++++++++ infra/otel/prometheus.yaml | 2 + infra/otel/runbook.md | 27 ++ scripts/check_observability.py | 33 +- scripts/check_observability_incidents.py | 171 ++++++++++ src/backend/app/server.go | 3 + src/backend/repositories/broker.go | 8 +- src/backend/repositories/database.go | 3 + src/backend/repositories/jobs.go | 31 +- .../000010_job_trace_context.down.sql | 1 + .../000010_job_trace_context.up.sql | 3 + src/backend/telemetry/capacity.go | 94 ++++++ src/backend/telemetry/postgres.go | 23 ++ src/backend/telemetry/propagation.go | 47 +++ src/backend/telemetry/telemetry.go | 21 +- src/backend/telemetry/worker.go | 9 +- src/backend/workers/pipeline.go | 35 +- 19 files changed, 821 insertions(+), 21 deletions(-) create mode 100644 infra/otel/alerts.yaml create mode 100644 infra/otel/runbook.md create mode 100644 scripts/check_observability_incidents.py create mode 100644 src/backend/repositories/migrations/000010_job_trace_context.down.sql create mode 100644 src/backend/repositories/migrations/000010_job_trace_context.up.sql create mode 100644 src/backend/telemetry/capacity.go create mode 100644 src/backend/telemetry/postgres.go create mode 100644 src/backend/telemetry/propagation.go diff --git a/compose.yml b/compose.yml index 4a65206..425cfb6 100644 --- a/compose.yml +++ b/compose.yml @@ -203,7 +203,7 @@ services: restart: unless-stopped cpus: 0.25 mem_limit: 128m - networks: [data] + networks: [data, telemetry] environment: *api-environment volumes: *api-volumes command: ["./messenger", "dispatcher"] @@ -334,6 +334,7 @@ services: command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] volumes: - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro - prometheus_data:/prometheus loki: diff --git a/infra/otel/alerts.yaml b/infra/otel/alerts.yaml new file mode 100644 index 0000000..5b13b46 --- /dev/null +++ b/infra/otel/alerts.yaml @@ -0,0 +1,20 @@ +groups: + - name: messenger + interval: 5s + rules: + - alert: MessengerHighErrorRate + expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05 + for: 15s + labels: + severity: warning + annotations: + summary: More than 5% of business requests return 5xx over one minute + runbook: Check API error spans and PostgreSQL operations in Tempo. + - alert: MessengerQueueLag + expr: max(messenger_queue_oldest_seconds) > 20 + for: 15s + labels: + severity: warning + annotations: + summary: Oldest outstanding attachment job is over 20 seconds old + runbook: Check outstanding jobs, worker saturation and consumer traces. diff --git a/infra/otel/grafana/dashboards/messenger.json b/infra/otel/grafana/dashboards/messenger.json index 5270463..a7cf8d0 100644 --- a/infra/otel/grafana/dashboards/messenger.json +++ b/infra/otel/grafana/dashboards/messenger.json @@ -200,6 +200,314 @@ }, "overrides": [] } + }, + { + "id": 7, + "title": "API · p50 длительности", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 8, + "title": "API · p99 длительности", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 9, + "title": "API · доля 5xx", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 10, + "title": "Очередь attachments.v1 · незавершённые задания", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 11, + "title": "Очередь attachments.v1 · возраст старейшего задания", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_seconds)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 12, + "title": "PostgreSQL · занятые соединения", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (messenger_db_pool_used)", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 13, + "title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "id": 14, + "title": "Worker · загрузка", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 15, + "title": "Вложения · p95 до ready", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 16, + "title": "Процессы · загрузка CPU", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + } + }, + { + "id": 17, + "title": "Процессы · резидентная память", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (messenger_process_rss_bytes)", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } } ] } diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml index 245214c..be8489e 100644 --- a/infra/otel/prometheus.yaml +++ b/infra/otel/prometheus.yaml @@ -1,5 +1,7 @@ global: scrape_interval: 5s +rule_files: + - /etc/prometheus/alerts.yaml scrape_configs: - job_name: otel-collector static_configs: diff --git a/infra/otel/runbook.md b/infra/otel/runbook.md new file mode 100644 index 0000000..8240fb1 --- /dev/null +++ b/infra/otel/runbook.md @@ -0,0 +1,27 @@ +# Диагностика API и обработки вложений + +SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка: + +```promql +1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m])) +``` + +Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул. + +`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert. + +Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go. + +Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны. + +Команды локальной проверки после `make app-up`: + +```sh +python3 scripts/check_observability.py pipeline +python3 scripts/check_observability_incidents.py worker +python3 scripts/check_observability_incidents.py database +python3 scripts/check_observability_incidents.py retry +python3 scripts/check_observability_incidents.py telemetry +``` + +Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение. diff --git a/scripts/check_observability.py b/scripts/check_observability.py index 8f7d994..be33a72 100644 --- a/scripts/check_observability.py +++ b/scripts/check_observability.py @@ -31,7 +31,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--grafana-url", default="http://127.0.0.1:3000") - parser.add_argument("criterion", choices=["provisioning", "signals"]) + parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"]) args = parser.parse_args() def query(path, **params): @@ -47,13 +47,42 @@ def main(): check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch") print("Источник Grafana:", uid) dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"] - check(len(dashboard["panels"]) == 6, "dashboard panels missing") + check(len(dashboard["panels"]) >= 6, "dashboard panels missing") print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"])) return client = Client(args.base_url) start = time.time_ns() alice, bob, _, chat = client.bootstrap() + if args.criterion == "pipeline": + upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", + 202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(), + headers={"Content-Type": "image/png"}, schema="Attachment") + client.wait_ready(bob, upload["json"]) + request_id = upload["headers"]["X-Request-Id"] + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + logs = source("loki", "/loki/api/v1/query_range", + query='{service_name="messenger-api"} | request_id="' + request_id + '"', + start=start, end=time.time_ns())["data"]["result"] + if logs: + trace_id = logs[0]["stream"]["trace_id"] + trace = source("tempo", "/api/traces/" + trace_id) + spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]] + producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None) + consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None) + if producer and consumer: + api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER") + check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch") + check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch") + children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]} + check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children, + "worker S3 and PostgreSQL spans missing") + print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id) + return + time.sleep(2) + raise Failure("upload trace did not include the worker pipeline") + marker = "telemetry-secret-" + uuid.uuid4().hex message = client.message(alice, chat, marker) request_id = message["headers"]["X-Request-Id"] diff --git a/scripts/check_observability_incidents.py b/scripts/check_observability_incidents.py new file mode 100644 index 0000000..648906b --- /dev/null +++ b/scripts/check_observability_incidents.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал.""" +import argparse +import json +from pathlib import Path +import re +import subprocess +import sys +import time +import urllib.parse +import urllib.request +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure + + +def compose(*args): + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"]) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--grafana-url", default="http://127.0.0.1:3000") + args = parser.parse_args() + client = Client(args.base_url) + alice, _, _, chat = client.bootstrap() + + def trace_for(request_id, matches): + base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-" + query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'}) + deadline = time.monotonic() + 35 + while time.monotonic() < deadline: + with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response: + logs = json.load(response)["data"]["result"] + if logs: + trace_id = logs[0]["stream"]["trace_id"] + try: + with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response: + trace = json.load(response) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + else: + spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]] + if matches(spans): + return trace_id + time.sleep(2) + raise Failure("expected trace was not found") + + def alerts(name): + endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts" + with urllib.request.urlopen(endpoint, timeout=5) as response: + active = json.load(response)["data"]["alerts"] + return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active) + + def wait_alert(name, firing, exercise=lambda: None): + deadline = time.monotonic() + 100 + while time.monotonic() < deadline: + exercise() + if alerts(name) == firing: + print(name, "firing" if firing else "resolved", flush=True) + return + time.sleep(2) + raise Failure(name + ": expected alert state not observed") + + if args.incident == "worker": + compose("stop", "worker") + try: + attachment, _ = client.attachment(alice, chat) + wait_alert("MessengerQueueLag", True) + finally: + compose("start", "worker") + client.wait_ready(alice, attachment) + wait_alert("MessengerQueueLag", False) + elif args.incident == "database": + # Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL. + marker = "lab6-fault-" + uuid.uuid4().hex + command = ["docker", "compose", "exec", "-T", "db", "sh", "-c", + 'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" ' + '-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"', + "sh", marker] + lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL) + try: + time.sleep(2) + + request_ids = [] + + def exercise(): + response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice) + request_ids.append(response["headers"]["X-Request-Id"]) + + wait_alert("MessengerHighErrorRate", True, exercise) + finally: + # Terminate only the synthetic lock session so the transaction rolls back immediately. + compose("exec", "-T", "db", "sh", "-c", + 'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + "WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'") + lock.wait(timeout=10) + client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat") + trace_id = trace_for(request_ids[-1], lambda spans: any( + span["name"] == "postgresql query" + and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans)) + print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True) + wait_alert("MessengerHighErrorRate", False) + elif args.incident == "retry": + compose("stop", "worker") + try: + upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", + 202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(), + headers={"Content-Type": "image/png"}, schema="Attachment") + attachment = upload["json"] + compose("stop", "s3") + try: + compose("up", "-d", "--no-deps", "worker") + deadline = time.monotonic() + 25 + while time.monotonic() < deadline: + sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id " + "WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'") + output = subprocess.check_output( + ["docker", "compose", "exec", "-T", "db", "sh", "-c", + 'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql], + cwd=ROOT, text=True, timeout=10) + if int(output.strip()) > 0: + break + time.sleep(1) + else: + raise Failure("durable retry was not observed") + finally: + compose("up", "-d", "--wait", "s3") + client.wait_ready(alice, attachment) + def retried(spans): + attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER" + for a in span.get("attributes", []) if a["key"] == "job.attempt"] + return len(set(attempts)) >= 2 + + trace_id = trace_for(upload["headers"]["X-Request-Id"], retried) + print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True) + finally: + compose("start", "worker") + else: + since = str(int(time.time())) + compose("stop", "tempo") + try: + for _ in range(8): + client.message(alice, chat, "telemetry-backend-off") + time.sleep(5) + attachment, _ = client.attachment(alice, chat) + client.wait_ready(alice, attachment) + compose("stats", "--no-stream", "otel-collector") + print("API и worker работали при остановленном Tempo.", flush=True) + finally: + compose("start", "tempo") + output = subprocess.check_output( + ["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"], + cwd=ROOT, text=True, timeout=15) + dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line + for match in re.findall(r'"dropped_items":\s*(\d+)', line)) + print("Спанов отброшено Collector за период проверки:", dropped, flush=True) + + +if __name__ == "__main__": + try: + main() + except (Failure, OSError, subprocess.SubprocessError, ValueError) as error: + print("FAIL:", error, file=sys.stderr) + sys.exit(1) diff --git a/src/backend/app/server.go b/src/backend/app/server.go index 28e08dc..bad0c28 100644 --- a/src/backend/app/server.go +++ b/src/backend/app/server.go @@ -94,6 +94,9 @@ func RunHTTP() error { return fmt.Errorf("telemetry initialization failed: %w", err) } defer stopTelemetry() + if err := signals.ObservePool(db); err != nil { + return err + } server := &http.Server{ Addr: serverAddress(), diff --git a/src/backend/repositories/broker.go b/src/backend/repositories/broker.go index f16e664..4704b8a 100644 --- a/src/backend/repositories/broker.go +++ b/src/backend/repositories/broker.go @@ -2,7 +2,9 @@ package repositories import ( "context" + "errors" + "messenger/src/backend/telemetry" "net" "time" @@ -53,8 +55,12 @@ func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() } func (b *JobBroker) Publish(ctx context.Context, body []byte) error { // mandatory catches an unroutable message; confirms alone do not prove routing. + headers := amqp.Table{} + for key, value := range telemetry.TraceHeaders(ctx) { + headers[key] = value + } confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{ - DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, + DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers, }) if err != nil { return errors.New("RabbitMQ publication failed") diff --git a/src/backend/repositories/database.go b/src/backend/repositories/database.go index 140e734..2234049 100644 --- a/src/backend/repositories/database.go +++ b/src/backend/repositories/database.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "messenger/src/backend/telemetry" + "github.com/golang-migrate/migrate/v4" pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5" "github.com/golang-migrate/migrate/v4/source/iofs" @@ -43,6 +45,7 @@ func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) { if err != nil { return nil, err } + cfg.ConnConfig.Tracer = telemetry.PostgresTracer{} cfg.MaxConns = 10 if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" { maxConns, parseErr := strconv.Atoi(value) diff --git a/src/backend/repositories/jobs.go b/src/backend/repositories/jobs.go index ef1d5ce..981dfbc 100644 --- a/src/backend/repositories/jobs.go +++ b/src/backend/repositories/jobs.go @@ -5,10 +5,14 @@ import ( "crypto/rand" "encoding/json" "errors" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" "math/big" "time" + + "messenger/src/backend/telemetry" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/trace" ) type Variant struct { @@ -27,12 +31,14 @@ type Job struct { PipelineVersion int `json:"pipeline_version"` Attempt int `json:"attempt"` CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"-"` + Tracestate string `json:"-"` } // QueuedJobs is also the recovery source when publish/confirm is uncertain. // Re-publication is intentional: a durable intent is never removed by publish. func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) { - rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until= 2 { + if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil { + observer.ObserveInt64(rss, pages*int64(os.Getpagesize())) + } + } + } + return nil + }, cpu, rss) + return err +} diff --git a/src/backend/telemetry/postgres.go b/src/backend/telemetry/postgres.go new file mode 100644 index 0000000..f22e157 --- /dev/null +++ b/src/backend/telemetry/postgres.go @@ -0,0 +1,23 @@ +package telemetry + +import ( + "context" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/trace" +) + +type PostgresTracer struct{} + +type queryFinishKey struct{} + +func (PostgresTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + ctx, finish := Operation(ctx, "postgresql query", trace.SpanKindClient) + return context.WithValue(ctx, queryFinishKey{}, finish) +} + +func (PostgresTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + if finish, ok := ctx.Value(queryFinishKey{}).(func(error)); ok { + finish(data.Err) + } +} diff --git a/src/backend/telemetry/propagation.go b/src/backend/telemetry/propagation.go new file mode 100644 index 0000000..bf09c49 --- /dev/null +++ b/src/backend/telemetry/propagation.go @@ -0,0 +1,47 @@ +package telemetry + +import ( + "context" + + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +func TraceHeaders(ctx context.Context) map[string]string { + headers := propagation.MapCarrier{} + propagation.TraceContext{}.Inject(ctx, headers) + return headers +} + +func WithTraceHeaders(ctx context.Context, headers map[string]string) context.Context { + return propagation.TraceContext{}.Extract(ctx, propagation.MapCarrier(headers)) +} + +func (s *Signals) Publish(ctx context.Context, publish func(context.Context) error) error { + if s == nil { + return publish(ctx) + } + ctx, span := s.tracer.Start(ctx, "attachments.v1 publish", trace.WithSpanKind(trace.SpanKindProducer)) + defer span.End() + err := publish(ctx) + if err != nil { + span.SetStatus(codes.Error, "") + } + return err +} + +// Operation creates a child of the active span without recording arguments or raw errors. +func Operation(ctx context.Context, name string, kind trace.SpanKind) (context.Context, func(error)) { + parent := trace.SpanFromContext(ctx) + if !parent.IsRecording() { + return ctx, func(error) {} + } + ctx, span := parent.TracerProvider().Tracer("messenger").Start(ctx, name, trace.WithSpanKind(kind)) + return ctx, func(err error) { + if err != nil { + span.SetStatus(codes.Error, "") + } + span.End() + } +} diff --git a/src/backend/telemetry/telemetry.go b/src/backend/telemetry/telemetry.go index 679953c..fb943a2 100644 --- a/src/backend/telemetry/telemetry.go +++ b/src/backend/telemetry/telemetry.go @@ -22,6 +22,9 @@ import ( ) type Signals struct { + meter metric.Meter + busy metric.Int64UpDownCounter + ready metric.Float64Histogram tracer trace.Tracer logger otellog.Logger requests metric.Int64Counter @@ -79,11 +82,25 @@ func Start(service string) (*Signals, func(), error) { httpDuration, errHTTP := meter.Float64Histogram("messenger.http.duration", metric.WithUnit("s"), buckets) jobs, errJobs := meter.Int64Counter("messenger.worker.jobs") jobDuration, errJobDuration := meter.Float64Histogram("messenger.worker.duration", metric.WithUnit("s"), buckets) - if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration); err != nil { + busy, errBusy := meter.Int64UpDownCounter("messenger.worker.busy") + capacity, errCapacity := meter.Int64UpDownCounter("messenger.worker.capacity") + ready, errReady := meter.Float64Histogram("messenger.attachment.ready", metric.WithUnit("s"), buckets) + if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration, errBusy, errCapacity, errReady); err != nil { stop() return nil, func() {}, err } - return &Signals{tp.Tracer("messenger"), lp.Logger("messenger"), requests, httpDuration, jobs, jobDuration}, stop, nil + if service == "messenger-worker" { + capacity.Add(ctx, 1) + busy.Add(ctx, 0) + } + signals := &Signals{meter: meter, busy: busy, ready: ready, tracer: tp.Tracer("messenger"), + logger: lp.Logger("messenger"), requests: requests, httpDuration: httpDuration, + jobs: jobs, jobDuration: jobDuration} + if err := signals.ObserveProcess(); err != nil { + stop() + return nil, func() {}, err + } + return signals, stop, nil } func (s *Signals) event(ctx context.Context, event string, failed bool, attrs ...otellog.KeyValue) { diff --git a/src/backend/telemetry/worker.go b/src/backend/telemetry/worker.go index 6d5a0c6..ece8478 100644 --- a/src/backend/telemetry/worker.go +++ b/src/backend/telemetry/worker.go @@ -11,14 +11,19 @@ import ( "go.opentelemetry.io/otel/trace" ) -func (s *Signals) Job(ctx context.Context, jobType, attachmentID string) (context.Context, func(string)) { +func (s *Signals) Job(ctx context.Context, jobType, attachmentID string, attempt int, accepted time.Time) (context.Context, func(string)) { if s == nil { return ctx, func(string) {} } start := time.Now() - ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer)) + s.busy.Add(ctx, 1) + ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(attribute.Int("job.attempt", attempt))) return ctx, func(outcome string) { defer span.End() + s.busy.Add(ctx, -1) + if outcome == "success" { + s.ready.Record(ctx, time.Since(accepted).Seconds()) + } attrs := []attribute.KeyValue{attribute.String("job.type", jobType), attribute.String("job.outcome", outcome)} span.SetAttributes(attrs...) failed := outcome == "error" diff --git a/src/backend/workers/pipeline.go b/src/backend/workers/pipeline.go index 96a23c0..e6c1707 100644 --- a/src/backend/workers/pipeline.go +++ b/src/backend/workers/pipeline.go @@ -11,6 +11,8 @@ import ( "syscall" "time" + "go.opentelemetry.io/otel/trace" + "messenger/src/backend/internal/faults" "messenger/src/backend/repositories" "messenger/src/backend/services" @@ -60,19 +62,28 @@ func RunPipeline(mode string, args []string) error { return err } var signals *telemetry.Signals - if mode == "worker" { + if mode == "worker" || mode == "dispatcher" { + service := "messenger-" + mode var stopTelemetry func() - signals, stopTelemetry, err = telemetry.Start("messenger-worker") + signals, stopTelemetry, err = telemetry.Start(service) if err != nil { return err } defer stopTelemetry() + if err := signals.ObservePool(db); err != nil { + return err + } + if mode == "dispatcher" { + if err := signals.ObserveBacklog(store.PendingJobs); err != nil { + return err + } + } } for ctx.Err() == nil { broker, e := repositories.NewJobBroker(ctx, address) if e == nil { if mode == "dispatcher" { - e = dispatchJobs(ctx, store, broker) + e = dispatchJobs(ctx, store, broker, signals) } else { e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout, signals) } @@ -89,7 +100,7 @@ func RunPipeline(mode string, args []string) error { return nil } -func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker) error { +func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker, signals *telemetry.Signals) error { for ctx.Err() == nil { jobs, err := store.QueuedJobs(ctx) if err != nil { @@ -102,7 +113,12 @@ func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, brok return err } publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - err = broker.Publish(publishCtx, payload) + publishCtx = telemetry.WithTraceHeaders(publishCtx, map[string]string{ + "traceparent": job.Traceparent, "tracestate": job.Tracestate, + }) + err = signals.Publish(publishCtx, func(ctx context.Context) error { + return broker.Publish(ctx, payload) + }) cancel() if err != nil { return err @@ -142,14 +158,19 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec continue } task, cancel := context.WithTimeout(ctx, timeout+15*time.Second) - task, finish := signals.Job(task, job.Type, job.AttachmentID) + parent, _ := message.Headers["traceparent"].(string) + state, _ := message.Headers["tracestate"].(string) + task = telemetry.WithTraceHeaders(task, map[string]string{"traceparent": parent, "tracestate": state}) + task, finish := signals.Job(task, job.Type, job.AttachmentID, job.Attempt, job.CreatedAt) var imageErr error token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts) if workErr == nil && token != "" { faults.Crash("before_effect", job.AttachmentID) faults.Pause(task, "after_claim", job.AttachmentID) workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { - variants, code, err := processImage(task, objects, a, staging) + imageCtx, finishImage := telemetry.Operation(task, "s3 read staging and write variants", trace.SpanKindClient) + variants, code, err := processImage(imageCtx, objects, a, staging) + finishImage(err) imageErr = err return variants, code, err })