From 44d81fd80ee13faa42bca069245ee0943f88b2df Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 12:57:10 +0300 Subject: [PATCH] Preserve distributed job context and expose grade 4 operational signals --- Makefile | 2 +- alembic/versions/0007_trace_context.py | 19 + .../dashboards/messenger-pipeline.json | 264 ++++++++++++- infra/prometheus/rules/messenger.alerts.yml | 30 ++ infra/prometheus/runbooks.md | 15 + .../.openspec.yaml | 2 + .../.openspec.yaml | 2 + .../design.md | 23 ++ .../proposal.md | 18 + .../lab6-observability-resilience/spec.md | 26 ++ .../tasks.md | 10 + scripts/obs_check.py | 1 + scripts/obs_incidents.py | 349 ++++++++++++++++++ .../attachments/application/pipeline.py | 2 +- .../attachments/application/ports.py | 9 +- src/messenger/attachments/entrypoints/ioc.py | 4 +- .../attachments/entrypoints/worker.py | 3 + .../attachments/infrastructure/postgres.py | 17 +- .../infrastructure/queue_metrics.py | 62 ++++ .../attachments/infrastructure/rabbitmq.py | 28 +- .../attachments/infrastructure/s3.py | 33 +- .../attachments/infrastructure/telemetry.py | 27 +- src/messenger/relay.py | 7 + .../shared/infrastructure/postgres.py | 33 +- .../shared/infrastructure/runtime_metrics.py | 26 ++ .../shared/infrastructure/telemetry.py | 2 + .../shared/infrastructure/trace_context.py | 22 ++ student-tests/api/test_bounded_contexts.py | 2 + student-tests/api/test_migrations.py | 9 +- student-tests/api/test_trace_context.py | 65 ++++ student-tests/pipeline.py | 8 +- student-tests/unit/test_job_observer.py | 13 + 32 files changed, 1098 insertions(+), 35 deletions(-) create mode 100644 alembic/versions/0007_trace_context.py create mode 100644 infra/prometheus/rules/messenger.alerts.yml create mode 100644 infra/prometheus/runbooks.md create mode 100644 openspec/changes/complete-lab6-observability-resilience/.openspec.yaml create mode 100644 openspec/changes/complete-lab6-observability-resilience/design.md create mode 100644 openspec/changes/complete-lab6-observability-resilience/proposal.md create mode 100644 openspec/changes/complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md create mode 100644 openspec/changes/complete-lab6-observability-resilience/tasks.md create mode 100644 scripts/obs_incidents.py create mode 100644 src/messenger/attachments/infrastructure/queue_metrics.py create mode 100644 src/messenger/shared/infrastructure/runtime_metrics.py create mode 100644 src/messenger/shared/infrastructure/trace_context.py create mode 100644 student-tests/api/test_trace_context.py diff --git a/Makefile b/Makefile index ecf62c25..3efc28f0 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMPOSE ?= docker compose RATE ?= 100 FROM_REV ?= SHIFT ?= 0 -SCRIPTS_PY = scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ +SCRIPTS_PY = scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \ scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py SCENARIO ?= failover-upload diff --git a/alembic/versions/0007_trace_context.py b/alembic/versions/0007_trace_context.py new file mode 100644 index 00000000..72be9f7f --- /dev/null +++ b/alembic/versions/0007_trace_context.py @@ -0,0 +1,19 @@ +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB + +revision = "0007_trace_context" +down_revision = "0006_attachment_quarantine" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + for table in ("jobs", "outbox"): + op.add_column( + table, sa.Column("trace_context", JSONB(), nullable=False, server_default="{}") + ) + + +def downgrade() -> None: + raise RuntimeError("Downgrade would delete trace correlation data") diff --git a/infra/grafana/dashboards/messenger-pipeline.json b/infra/grafana/dashboards/messenger-pipeline.json index c23dc39d..e34c7550 100644 --- a/infra/grafana/dashboards/messenger-pipeline.json +++ b/infra/grafana/dashboards/messenger-pipeline.json @@ -12,7 +12,7 @@ "graphTooltip": 1, "refresh": "10s", "schemaVersion": 41, - "version": 1, + "version": 2, "time": { "from": "now-30m", "to": "now" @@ -302,6 +302,268 @@ "w": 12, "h": 10 } + }, + { + "id": 16, + "type": "timeseries", + "title": "Durable queue depth (includes retry)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth)", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Oldest durable job age", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_age_seconds)", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 18, + "type": "timeseries", + "title": "DB pool used / waiting / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (__name__,job) ({__name__=~\"messenger_db_pool_(used|waiting|capacity)\"})", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 19, + "type": "timeseries", + "title": "Worker busy / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 28, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (__name__) ({__name__=~\"messenger_worker_(busy|capacity)\"})", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Process CPU cores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 36, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (job) (rate(process_cpu_time_seconds_total{job=~\"messenger-.*\"}[1m]))", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 21, + "type": "timeseries", + "title": "Process resident memory", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 36, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (job) (process_memory_usage_bytes{job=~\"messenger-.*\"})", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "id": 22, + "type": "timeseries", + "title": "Time to ready p50 / p95 / p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 44, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "0.5" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "0.95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "0.99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 23, + "type": "timeseries", + "title": "Ready within 30s, SLO 95% / 1h", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 44, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:ready_under_30s:ratio1h", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 24, + "type": "timeseries", + "title": "API availability, SLO 99% / 1h", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 52, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:availability:ratio1h", + "legendFormat": "{{job}} {{__name__}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } } ] } diff --git a/infra/prometheus/rules/messenger.alerts.yml b/infra/prometheus/rules/messenger.alerts.yml new file mode 100644 index 00000000..4f1720be --- /dev/null +++ b/infra/prometheus/rules/messenger.alerts.yml @@ -0,0 +1,30 @@ +groups: + - name: messenger-slo + interval: 5s + rules: + - record: messenger:http_errors:ratio_rate1m + expr: | + (sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*",http_response_status_code=~"5.."}[1m])) or vector(0)) + / clamp_min(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*"}[1m])), 0.001) + - record: messenger:availability:ratio1h + expr: 1 - messenger:http_errors:ratio_rate1h + - record: messenger:ready_under_30s:ratio1h + expr: sum(increase(messenger_job_time_to_ready_seconds_bucket{le="30"}[1h])) / clamp_min(sum(increase(messenger_job_time_to_ready_seconds_count[1h])), 1) + - alert: MessengerHighErrorRate + expr: messenger:http_errors:ratio_rate1m > 0.05 + for: 15s + labels: + severity: warning + annotations: + summary: More than 5% business HTTP 5xx over 1 minute + description: Availability target is 99% over 1 hour; health and client 4xx are excluded from failures. + runbook: infra/prometheus/runbooks.md#messengerhigherrorrate + - alert: MessengerQueueLag + expr: max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) > 15 and max(messenger_queue_depth{queue_name="attachments.process"}) > 0 + for: 15s + labels: + severity: warning + annotations: + summary: Durable attachment backlog has waited more than 15 seconds + description: Target is 95% ready within 30 seconds over 1 hour; depth includes retry and leased work. + runbook: infra/prometheus/runbooks.md#messengerqueuelag diff --git a/infra/prometheus/runbooks.md b/infra/prometheus/runbooks.md new file mode 100644 index 00000000..f24cb961 --- /dev/null +++ b/infra/prometheus/runbooks.md @@ -0,0 +1,15 @@ +# Локальная диагностика + +SLI availability = 1 − business HTTP 5xx / все business HTTP за 1 h, цель 99%; health исключены, 4xx остаются в знаменателе. SLI ready = доля завершённых jobs с time-to-ready <=30 s за 1 h, цель 95%. Незавершённые jobs не входят в histogram, поэтому возраст/depth контролируются отдельно. Учебные alerts реагируют раньше окна SLO: это пороговые предупреждения, не multi-window burn rates. + +## MessengerHighErrorRate + +Условие: 5xx/все >5% за 1 m непрерывно 15 s; вычисление каждые 5 s. В API dashboard проверить error ratio и p95. По request_id найти лог и trace. Длинный/ошибочный PostgreSQL client span указывает на зависимость; длинный server span без медленных client spans — на handler. Проверить used/waiting/capacity пула и ресурсы процесса. В своём стенде восстановить postgres, дождаться ready обеих API и успешных запросов. Alert должен исчезнуть после ухода ошибок из минутного окна. + +## MessengerQueueLag + +Условие: durable depth>0 и oldest age>15 s непрерывно 15 s. Проверить jobs outcomes, worker busy/capacity, CPU/RAM и наличие worker logs. Отсутствие worker signal при растущем backlog указывает на остановку процесса; retry с S3 ERROR spans — на хранилище. Проверить producer/consumer цепочку конкретного attachment. Восстановить только свой worker/зависимость; дождаться ready, depth=0, age=0 и отсутствия firing alert. Queue depth относится к PostgreSQL queued/processing, включая scheduled retry, а не только AMQP ready messages. + +## Отказ telemetry backend + +Readiness API не зависит от Collector/хранилищ. Сначала проверить бизнес-smoke. Collector: queue_size/capacity, send_failed и sent counters; Docker RAM против mem_limit. Retry ограничен 60 s, очереди ограничены; потеря диагностических данных при заполнении допустима, потеря сообщений — нет. Восстановить backend и проверить новый request trace/log и метрики. Не удалять volumes для восстановления. diff --git a/openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/.openspec.yaml b/openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/.openspec.yaml index 5c99d6d7..ca6cf43d 100644 --- a/openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/.openspec.yaml +++ b/openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/.openspec.yaml @@ -1,2 +1,4 @@ schema: spec-driven created: 2026-10-29 +archived: 2026-11-06 +lab: 6 diff --git a/openspec/changes/complete-lab6-observability-resilience/.openspec.yaml b/openspec/changes/complete-lab6-observability-resilience/.openspec.yaml new file mode 100644 index 00000000..8a94d769 --- /dev/null +++ b/openspec/changes/complete-lab6-observability-resilience/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-06 diff --git a/openspec/changes/complete-lab6-observability-resilience/design.md b/openspec/changes/complete-lab6-observability-resilience/design.md new file mode 100644 index 00000000..725b45bd --- /dev/null +++ b/openspec/changes/complete-lab6-observability-resilience/design.md @@ -0,0 +1,23 @@ +## Context + +SQLAlchemy async, transactional outbox и manual ack после COMMIT уже работают. SDK экспортирует асинхронно, Collector имеет memory_limiter, batch, ограниченные очереди и retry 60 s. + +## Goals / Non-Goals + +Закрыть все пункты уровня 4; не реализовывать трек 5 в этом change и не менять контракт API. + +## Decisions + +W3C traceparent/tracestate хранятся отдельно от нормативного payload в JSONB job и outbox. При retry берётся активный consumer context, при восстановлении аренды — сохранённый origin. Миграция добавляет поля с пустым default; прежние jobs остаются пригодными. Relay извлекает контекст и FastStream создаёт producer spans; ack остаётся после бизнес-COMMIT. S3 spans содержат только операцию/сервис, без ключей и exception payload. + +Queue depth/age измеряют durable backlog queued/processing PostgreSQL, включая retry: это не размер AMQP очереди. Pool used/waiting/capacity измеряются у SQLAlchemy engine и в ObservedPool._do_get, без идентификаторов. Worker busy/capacity отражают конкурентность; time-to-ready отсчитывается от created_at job до done. CPU — процессорные секунды, RAM — текущий RSS из /proc. Все gauges отдаются OTLP; статические labels и resource service instance. Histogram seconds использует границы 0.05…60, включая SLO 30 s. + +Availability SLO: >=99% успешных бизнес-HTTP за 1 h; ready SLO: >=95% done за 30 s на окне 1 h. Alerts: error ratio >5% за 1 m, for 15 s; backlog age >15 s, for 15 s. Thresholds короткие для воспроизводимой учебной демонстрации, не производственные рекомендации. Runbooks лежат в Git; внешняя доставка не требуется. + +## Risks / Trade-offs + +ObservedPool оборачивает получение connection, сохраняя ленивые sessions, timeout, autocommit и границы UoW. Waiting включает получение/создание соединения; используется закреплённая версия SQLAlchemy. Poller очереди работает отдельно, ошибки измерений не останавливают relay. Нулевой backlog возвращает age 0. При telemetry outage потеря сигналов допустима и измеряется; успешность сообщений обязательна. Проверки последовательны на собственном project, состояние Docker записывается; постоянные volumes не удаляются. + +## Validation + +Миграции legacy/fresh, тесты durable context и retry, make check/validate/lint/student-test. Реальные API→S3/DB→producer→consumer traces, retry, строгие dashboards, leak checks. Два инцидента с временной шкалой, queries и trace IDs, firing/resolved. Отказ хранилищ: smoke, RAM samples, Collector counters до/после, восстановление. Финальный REPORT делает следующий агент. diff --git a/openspec/changes/complete-lab6-observability-resilience/proposal.md b/openspec/changes/complete-lab6-observability-resilience/proposal.md new file mode 100644 index 00000000..0ad8757c --- /dev/null +++ b/openspec/changes/complete-lab6-observability-resilience/proposal.md @@ -0,0 +1,18 @@ +## Why + +Уровень 3 доказан. Сейчас outbox теряет контекст HTTP, relay не создаёт producer spans, а наблюдаемость не объясняет накопление очереди и зависание БД. Это обязательные критерии уровня 4. + +## What Changes + +- Добавочная Alembic-миграция сохраняет W3C контекст job/outbox, FastStream связывает API, producer и consumer; S3 получает безопасные client spans. +- Добавляются queue age/depth, pool used/waiting, worker busy/capacity, time-to-ready и process CPU/RAM, dashboards и SLI/SLO с alerts. +- Реальные инциденты worker stop и PostgreSQL outage подтверждают firing/resolved; отдельный отказ хранилищ проверяет доступность API и ограничение памяти. + +## Capabilities + +### New Capabilities +- `lab6-observability-resilience`: сквозная диагностика и устойчивость наблюдаемости уровня 4. + +## Impact + +Меняются собственные адаптеры, миграции, инструменты и provisioning. API, нормативные файлы и REPORT не меняются. Следующему агенту передаются evidence и заметки. Данные существующих таблиц сохраняются. diff --git a/openspec/changes/complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md b/openspec/changes/complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md new file mode 100644 index 00000000..43afff14 --- /dev/null +++ b/openspec/changes/complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md @@ -0,0 +1,26 @@ +## ADDED Requirements + +### Requirement: Durable distributed context +The system SHALL preserve W3C context through API, transactional outbox, relay producer and worker consumer including retries, and expose safe database and S3 client spans. + +#### Scenario: Deferred delivery and retry +- **WHEN** an accepted upload is delivered after relay restart or retried after S3 failure +- **THEN** its producer and consumer spans share the originating trace and causally valid parents, without adding fields to the normative job payload + +### Requirement: Operational signals and SLO +The system SHALL expose queue depth and oldest age, database pool usage and waiting, worker saturation, process CPU/RAM and durable time-to-ready; dashboards SHALL show histogram percentiles and error ratio and define availability and ready SLO windows. + +#### Scenario: Stopped worker +- **WHEN** queued uploads wait for a stopped worker +- **THEN** depth and age increase, a queue-lag alert fires and resolves after recovery + +#### Scenario: Unavailable database +- **WHEN** PostgreSQL becomes unavailable +- **THEN** API failure metrics and database client spans distinguish the dependency failure and an error-rate alert fires and resolves + +### Requirement: Bounded observability failure +Collector SHALL use memory limits, batching and bounded queues; unavailable telemetry storage SHALL NOT prevent business requests or cause unbounded memory growth. + +#### Scenario: Storage outage +- **WHEN** telemetry storage is stopped during requests +- **THEN** business smoke passes, measured memory remains within container limits, losses are recorded, and exports resume after recovery diff --git a/openspec/changes/complete-lab6-observability-resilience/tasks.md b/openspec/changes/complete-lab6-observability-resilience/tasks.md new file mode 100644 index 00000000..391d8bfa --- /dev/null +++ b/openspec/changes/complete-lab6-observability-resilience/tasks.md @@ -0,0 +1,10 @@ +## Implementation +- [ ] 1. Durable context migration, repository and publisher wiring; propagation/retry tests +- [ ] 2. Safe S3 spans and process/pool/queue/worker/time-to-ready metrics; unit checks +- [ ] 3. Dashboards, SLO recording/alert rules and runbooks; configuration validation +## Verification +- [ ] 4. make check/validate/lint/student-test; real smoke, persistence and failover after changes +- [ ] 5. End-to-end and retry traces, dashboards, leaks and cardinality +- [ ] 6. Worker and database incidents: evidence, firing/resolved and diagnosis +- [ ] 7. Storage outage: availability, bounded memory, lost signals and recovery +- [ ] 8. Evidence notes, requirement sync and archive after all checks diff --git a/scripts/obs_check.py b/scripts/obs_check.py index 598a0faa..02e78f37 100755 --- a/scripts/obs_check.py +++ b/scripts/obs_check.py @@ -131,6 +131,7 @@ def tempo_spans(body): duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6 spans.append({"service": service, "name": span["name"], "kind": span.get("kind"), "status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"), + "span_id": span.get("spanId"), "parent_span_id": span.get("parentSpanId"), "duration_ms": round(duration, 3), "links": len(span.get("links", [])), "attributes": values}) return spans diff --git a/scripts/obs_incidents.py b/scripts/obs_incidents.py new file mode 100644 index 00000000..8399c737 --- /dev/null +++ b/scripts/obs_incidents.py @@ -0,0 +1,349 @@ +import argparse +import concurrent.futures +import json +import os +import subprocess +import time +import uuid +from pathlib import Path + +from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for +from obs_recovery import backends, compose + + +def actor(): + api = Api(os.environ["BASE_URL"]) + user, username, password, _ = api.register() + auth, _ = api.login(username, password) + return api, auth, api.chat(auth, user) + + +def upload(api, auth, chat): + return api.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png", + Path("tests/fixtures/sample.png").read_bytes(), + dict(auth, **{"Content-Type": "image/png"}), + expect=(202,), + ) + + +def ready(api, auth, attachment): + def check(): + item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0] + if item["status"] == "failed": + raise RuntimeError("attachment failed") + return item if item["status"] == "ready" else None + + return wait_for("ready", check, 120, 1) + + +def value(b, expression): + result = b.promql(expression) + return float(result[0]["value"][1]) if result else None + + +def alert(b, name, firing): + def check(): + alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"] + found = [ + a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing" + ] + if bool(found) == firing: + return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found} + return None + + return wait_for(name, check, 150) + + +def chain(b, retry): + api, auth, chat = actor() + started = time.time() + if retry: + compose("stop", "worker") + attachment, rid = upload(api, auth, chat) + try: + if retry: + compose("stop", "s3") + compose("start", "worker") + query = ( + '{service_name="messenger-worker"} | attachment_id="' + + attachment["id"] + + '" | outcome="retry_scheduled"' + ) + wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60) + finally: + if retry: + compose("up", "-d", "--no-deps", "--wait", "s3", "worker") + ready(api, auth, attachment) + logs = wait_for( + "upload log", + lambda: b.logs( + '{service_name="messenger-api"} | request_id="' + rid + '"', + started - 1, + time.time() + 1, + ), + 60, + ) + trace_id = logs[0][0]["trace_id"] + + def complete(): + spans = tempo_spans(b.trace(trace_id)[0]) + return ( + spans + if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans) + else None + ) + + spans = wait_for("complete trace", complete, 90) + roots = [ + s + for s in spans + if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid + ] + ids = {s["span_id"] for s in spans} + producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"] + consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"] + clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"] + if ( + not roots + or not producers + or not consumers + or any(s["parent_span_id"] not in ids for s in producers + consumers) + ): + raise Failure("broken distributed parent chain") + if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any( + s["attributes"].get("db.system") == "postgresql" for s in clients + ): + raise Failure("missing dependency spans") + outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers] + if retry and "retry_scheduled" not in outcomes: + raise Failure("retry left original trace") + api.logout(auth) + fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms") + return { + "upload_request_id": rid, + "trace_id": trace_id, + "retry": retry, + "outcomes": outcomes, + "spans": [ + {**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")} + for s in spans + ], + } + + +def worker(b): + api, auth, chat = actor() + compose("stop", "worker") + stopped = now() + try: + items = [upload(api, auth, chat)[0] for _ in range(8)] + firing = alert(b, "MessengerQueueLag", True) + observed = { + "depth": value(b, "max(messenger_queue_depth)"), + "age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"), + } + finally: + compose("up", "-d", "--no-deps", "--wait", "worker") + restored = now() + for item in items: + ready(api, auth, item) + resolved = alert(b, "MessengerQueueLag", False) + api.logout(auth) + return { + "stopped": stopped, + "restored": restored, + "jobs": len(items), + "backlog": observed, + "firing": firing, + "resolved": resolved, + "depth_after": value(b, "max(messenger_queue_depth)"), + } + + +def database(b): + api, auth, _ = actor() + started = time.time() + prefix = "db-incident-" + uuid.uuid4().hex[:12] + + def request(index): + rid = prefix + "-" + str(index) + status = http( + "GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid}) + )[0] + return {"request_id": rid, "status": status} + + for index in range(10): + request(index) + time.sleep(12) + compose("pause", "postgres") + paused = now() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + responses = list(pool.map(request, range(10, 34))) + firing = alert(b, "MessengerHighErrorRate", True) + signals = { + "error_ratio": value(b, "messenger:http_errors:ratio_rate1m"), + "pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"), + } + finally: + compose("unpause", "postgres") + restored = now() + wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90) + for index in range(34, 74): + request(index) + time.sleep(0.15) + resolved = alert(b, "MessengerHighErrorRate", False) + traces = b.traceql( + '{ span.messenger.request_id =~ "' + prefix + '.*" }', + started - 1, + time.time() + 1, + limit=100, + ) + dependency = [] + for trace in traces: + spans = tempo_spans(b.trace(trace["traceID"])[0]) + failed = [ + s + for s in spans + if s["attributes"].get("db.system") == "postgresql" + and s["status"] == "STATUS_CODE_ERROR" + ] + if failed: + dependency.append( + { + "trace_id": trace["traceID"], + "db_errors": [ + {k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed + ], + } + ) + if not dependency or not any(r["status"] == 503 for r in responses): + raise Failure("database incident has no dependency failure trace") + api.logout(auth) + return { + "paused": paused, + "restored": restored, + "requests": responses, + "signals": signals, + "firing": firing, + "resolved": resolved, + "dependency_traces": dependency, + } + + +def collector_metrics(): + source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())' + raw = compose("exec", "-T", "api", "python", "-c", source) + return { + line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1]) + for line in raw.splitlines() + if line.startswith( + ( + "otelcol_receiver_accepted_", + "otelcol_exporter_sent_", + "otelcol_exporter_send_failed_", + "otelcol_exporter_queue_", + ) + ) + } + + +def memory(): + ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines() + raw = subprocess.check_output( + ["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True + ) + return [ + {k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")} + for row in map(json.loads, raw.splitlines()) + ] + + +def storage(b): + api, auth, _ = actor() + before = collector_metrics() + samples = [{"utc": now(), "containers": memory()}] + compose("stop", "prometheus", "loki", "tempo") + stopped = now() + requests = [] + try: + result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True) + until = time.monotonic() + 85 + while time.monotonic() < until: + rid = "storage-outage-" + uuid.uuid4().hex + status = http( + "GET", + api.base + "/api/v1/users/me", + headers=dict(auth, **{"X-Request-Id": rid}), + expect=(200,), + )[0] + requests.append({"id": rid, "status": status}) + if len(requests) % 15 == 0: + samples.append({"utc": now(), "containers": memory()}) + time.sleep(1) + during = collector_metrics() + finally: + compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo") + restored = now() + time.sleep(20) + after = collector_metrics() + samples.append({"utc": now(), "containers": memory()}) + found = 0 + for item in requests: + found += bool( + b.traceql( + '{ span.messenger.request_id = "' + item["id"] + '" }', + time.time() - 300, + time.time() + 1, + ) + ) + peak_percent = max( + float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"] + ) + if peak_percent >= 100: + raise Failure("container memory limit reached") + api.logout(auth) + return { + "stopped": stopped, + "restored": restored, + "peak_memory_percent_of_limit": peak_percent, + "smoke": result.stdout, + "successful_probes": len(requests), + "stored_probe_traces": found, + "lost_probe_traces": len(requests) - found, + "memory_samples": samples, + "collector_before": before, + "collector_during": during, + "collector_after": after, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage")) + args = parser.parse_args() + b = backends() + started = now() + report = ( + chain(b, args.scenario == "retry") + if args.scenario in ("chain", "retry") + else {"worker": worker, "database": database, "storage": storage}[args.scenario](b) + ) + print( + json.dumps( + { + "started": started, + "finished": now(), + "result": "PASS", + "scenario": args.scenario, + "observations": report, + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py index 8d51e345..bd518af3 100644 --- a/src/messenger/attachments/application/pipeline.py +++ b/src/messenger/attachments/application/pipeline.py @@ -223,7 +223,7 @@ class PublishOutboxHandler: for entry in await self._uow.outbox.pending(command.limit): await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH) try: - await self._publisher.publish(entry.job_id, entry.body) + await self._publisher.publish(entry.job_id, entry.body, entry.trace_context) except Unavailable as error: failure = error break diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index 839401d9..cde86fb1 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -1,6 +1,6 @@ from collections.abc import AsyncIterator, Sequence from contextlib import AbstractAsyncContextManager -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta from typing import Protocol, runtime_checkable from uuid import UUID @@ -33,6 +33,7 @@ class OutboxEntry: id: int job_id: UUID body: bytes + trace_context: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True, slots=True) @@ -77,7 +78,9 @@ class VariantRenderer(Protocol): @runtime_checkable class JobPublisher(Protocol): - async def publish(self, job_id: UUID, body: bytes) -> None: ... + async def publish( + self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None + ) -> None: ... @runtime_checkable @@ -179,6 +182,8 @@ class AttachmentDirectory(Protocol): @runtime_checkable class JobObserver(Protocol): + def active(self, delta: int) -> None: ... + def observe(self, job: Job, outcome: str, seconds: float) -> None: ... diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index d3480197..f06f62dd 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -203,8 +203,8 @@ class WorkerProvider(Provider): return RandomJitter() @provide(scope=Scope.APP) - def observer(self) -> JobObserver: - return OtelJobObserver() + def observer(self, settings: PipelineSettings) -> JobObserver: + return OtelJobObserver(capacity=settings.worker_concurrency) @provide(scope=Scope.REQUEST) def command_bus( diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py index f6ecb44c..a60f955e 100644 --- a/src/messenger/attachments/entrypoints/worker.py +++ b/src/messenger/attachments/entrypoints/worker.py @@ -35,6 +35,7 @@ async def handle_job( logger.warning("job.rejected", reason="message does not match jobs.schema.json") await message.reject(requeue=False) return + observer.active(1) started = time.perf_counter() try: outcome = (await bus.execute(ProcessJob(job))).value @@ -49,6 +50,8 @@ async def handle_job( await asyncio.sleep(REQUEUE_DELAY_SECONDS) await message.nack(requeue=True) return + finally: + observer.active(-1) observed(job, outcome, started, observer, logger) await message.ack() diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index 451995c4..a0e75ea7 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -45,6 +45,7 @@ from messenger.shared.infrastructure.postgres import ( EventSource, PostgresUnitOfWork, ) +from messenger.shared.infrastructure.trace_context import capture_context class AttachmentRow(Base): @@ -153,6 +154,7 @@ class JobRow(Base): pipeline_version: Mapped[int] status: Mapped[str] attempt: Mapped[int] + trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}") claims: Mapped[int] = mapped_column(server_default="0") lease_token: Mapped[UUID | None] lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) @@ -168,6 +170,7 @@ class OutboxRow(Base): id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True) job_id: Mapped[UUID] = mapped_column(ForeignKey("jobs.job_id")) payload: Mapped[dict[str, Any]] = mapped_column(JSONB) + trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}") available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) @@ -472,10 +475,18 @@ class PostgresJobRepository: self._session = session async def _enqueue(self, job: Job, available_at: datetime | None = None) -> None: + context = ( + capture_context() + or await self._session.scalar( + select(JobRow.trace_context).where(JobRow.job_id == job.id) + ) + or {} + ) await self._session.execute( insert(OutboxRow).values( job_id=job.id, payload=job.payload(), + trace_context=context, available_at=func.now() if available_at is None else available_at, ) ) @@ -518,6 +529,7 @@ class PostgresJobRepository: attachment_id=job.attachment_id, pipeline_version=job.pipeline_version, status="queued", + trace_context=capture_context(), attempt=job.attempt, created_at=job.created_at, updated_at=job.created_at, @@ -824,7 +836,10 @@ class PostgresOutboxRepository: .limit(limit) .with_for_update(skip_locked=True) ) - return [OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode()) for row in rows] + return [ + OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode(), row.trace_context) + for row in rows + ] async def published(self, entry_ids: Sequence[int]) -> None: await self._session.execute( diff --git a/src/messenger/attachments/infrastructure/queue_metrics.py b/src/messenger/attachments/infrastructure/queue_metrics.py new file mode 100644 index 00000000..bc206d73 --- /dev/null +++ b/src/messenger/attachments/infrastructure/queue_metrics.py @@ -0,0 +1,62 @@ +import asyncio +from contextlib import suppress +from datetime import UTC, datetime + +from opentelemetry import metrics +from opentelemetry.metrics import Observation +from sqlalchemy import func, select + +from messenger.attachments.infrastructure.postgres import JobRow +from messenger.shared.application.ports import Logger +from messenger.shared.infrastructure.postgres import Database + + +class QueueMetrics: + __slots__ = ("_database", "_depth", "_logger", "_oldest", "_task") + + def __init__(self, database: Database, logger: Logger) -> None: + self._database = database + self._logger = logger + self._depth = 0 + self._oldest: datetime | None = None + self._task: asyncio.Task[None] | None = None + meter = metrics.get_meter("messenger.queue") + labels = {"queue.name": "attachments.process"} + meter.create_observable_gauge( + "messenger.queue.depth", + unit="{job}", + callbacks=[lambda _: [Observation(self._depth, labels)]], + ) + meter.create_observable_gauge( + "messenger.queue.oldest_age", + unit="s", + callbacks=[lambda _: [Observation(self.age(), labels)]], + ) + + def age(self) -> float: + return max(0, (datetime.now(UTC) - self._oldest).total_seconds()) if self._oldest else 0 + + def start(self) -> None: + self._task = asyncio.create_task(self._poll()) + + async def stop(self) -> None: + if self._task: + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + + async def _poll(self) -> None: + while True: + try: + async with self._database.session() as session: + row = ( + await session.execute( + select(func.count(), func.min(JobRow.created_at)).where( + JobRow.status.in_(("queued", "processing")) + ) + ) + ).one() + self._depth, self._oldest = row + except Exception as error: + self._logger.warning("queue.metrics_failed", error=type(error).__name__) + await asyncio.sleep(5) diff --git a/src/messenger/attachments/infrastructure/rabbitmq.py b/src/messenger/attachments/infrastructure/rabbitmq.py index 778c1af9..428dda26 100644 --- a/src/messenger/attachments/infrastructure/rabbitmq.py +++ b/src/messenger/attachments/infrastructure/rabbitmq.py @@ -5,6 +5,7 @@ from pamqp.commands import Basic from messenger.attachments.domain.jobs import JOB_TYPE from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.trace_context import restored_context EXCHANGE = RabbitExchange("attachments", declare=False) QUEUE = "attachments.process" @@ -18,19 +19,22 @@ class RabbitJobPublisher: self._broker = broker self._timeout = timeout - async def publish(self, job_id: UUID, body: bytes) -> None: + async def publish( + self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None + ) -> None: try: - confirmation = await self._broker.publish( - body, - exchange=EXCHANGE, - routing_key=JOB_TYPE, - mandatory=True, - persist=True, - message_id=str(job_id), - message_type=JOB_TYPE, - content_type="application/json", - timeout=self._timeout, - ) + with restored_context(trace_context): + confirmation = await self._broker.publish( + body, + exchange=EXCHANGE, + routing_key=JOB_TYPE, + mandatory=True, + persist=True, + message_id=str(job_id), + message_type=JOB_TYPE, + content_type="application/json", + timeout=self._timeout, + ) except Exception as error: raise Unavailable("брокер не принял задание") from error if not isinstance(confirmation, Basic.Ack): diff --git a/src/messenger/attachments/infrastructure/s3.py b/src/messenger/attachments/infrastructure/s3.py index e71a82b0..413a35e8 100644 --- a/src/messenger/attachments/infrastructure/s3.py +++ b/src/messenger/attachments/infrastructure/s3.py @@ -6,6 +6,8 @@ from aiobotocore.session import get_session from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from botocore.handlers import add_expect_header +from opentelemetry import trace +from opentelemetry.trace import SpanKind, StatusCode from messenger.attachments.application.errors import ObjectMissingError from messenger.attachments.application.ports import ObjectContent, StoredObject @@ -60,17 +62,31 @@ class S3ObjectStorage: self._client = client self._bucket = bucket + async def _call(self, operation: str, **arguments: Any) -> Any: + with trace.get_tracer(__name__).start_as_current_span( + f"S3 {operation}", + kind=SpanKind.CLIENT, + attributes={"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": operation}, + record_exception=False, + set_status_on_exception=False, + ) as span: + try: + return await getattr(self._client, operation)(**arguments) + except Exception as error: + span.set_status(StatusCode.ERROR, type(error).__name__) + raise + async def put(self, key: str, data: bytes, content_type: str) -> None: try: - await self._client.put_object( - Bucket=self._bucket, Key=key, Body=data, ContentType=content_type + await self._call( + "put_object", Bucket=self._bucket, Key=key, Body=data, ContentType=content_type ) except FAILURES as error: raise Unavailable("S3 недоступно") from error async def open(self, key: str) -> ObjectContent: try: - response = await self._client.get_object(Bucket=self._bucket, Key=key) + response = await self._call("get_object", Bucket=self._bucket, Key=key) except FAILURES as error: if missing(error): raise ObjectMissingError(key) from error @@ -86,15 +102,18 @@ class S3ObjectStorage: async def copy(self, source: str, target: str) -> None: try: - await self._client.copy_object( - Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source} + await self._call( + "copy_object", + Bucket=self._bucket, + Key=target, + CopySource={"Bucket": self._bucket, "Key": source}, ) except FAILURES as error: raise Unavailable("S3 недоступно") from error async def delete(self, key: str) -> None: try: - await self._client.delete_object(Bucket=self._bucket, Key=key) + await self._call("delete_object", Bucket=self._bucket, Key=key) except FAILURES as error: raise Unavailable("S3 недоступно") from error @@ -113,7 +132,7 @@ class S3ObjectStorage: async def stat(self, key: str) -> StoredObject | None: try: - head = await self._client.head_object(Bucket=self._bucket, Key=key) + head = await self._call("head_object", Bucket=self._bucket, Key=key) except FAILURES as error: if missing(error): return None diff --git a/src/messenger/attachments/infrastructure/telemetry.py b/src/messenger/attachments/infrastructure/telemetry.py index 7a6a2f2b..c6741104 100644 --- a/src/messenger/attachments/infrastructure/telemetry.py +++ b/src/messenger/attachments/infrastructure/telemetry.py @@ -1,5 +1,7 @@ +from datetime import UTC, datetime + from opentelemetry import metrics, trace -from opentelemetry.metrics import MeterProvider +from opentelemetry.metrics import MeterProvider, Observation from opentelemetry.trace import StatusCode from messenger.attachments.domain.jobs import JOB_TYPE, Job @@ -9,10 +11,22 @@ FAILURES = frozenset({"retry_scheduled", "failed", "error"}) class OtelJobObserver: - __slots__ = ("_duration", "_processed") + __slots__ = ("_busy", "_duration", "_processed", "_ready") - def __init__(self, provider: MeterProvider | None = None) -> None: + def __init__(self, provider: MeterProvider | None = None, capacity: int = 1) -> None: meter = metrics.get_meter("messenger.attachments", meter_provider=provider) + self._busy = 0 + meter.create_observable_gauge( + "messenger.worker.busy", callbacks=[lambda _: [Observation(self._busy)]], unit="{job}" + ) + meter.create_observable_gauge( + "messenger.worker.capacity", callbacks=[lambda _: [Observation(capacity)]], unit="{job}" + ) + self._ready = meter.create_histogram( + "messenger.job.time_to_ready", + unit="s", + explicit_bucket_boundaries_advisory=DURATION_BUCKETS, + ) self._processed = meter.create_counter( "messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу" ) @@ -23,7 +37,14 @@ class OtelJobObserver: explicit_bucket_boundaries_advisory=DURATION_BUCKETS, ) + def active(self, delta: int) -> None: + self._busy += delta + def observe(self, job: Job, outcome: str, seconds: float) -> None: + if outcome == "done": + self._ready.record( + max(0, (datetime.now(UTC) - job.created_at).total_seconds()), {"job.type": JOB_TYPE} + ) labels = {"job.type": JOB_TYPE, "outcome": outcome} self._processed.add(1, labels) self._duration.record(seconds, labels) diff --git a/src/messenger/relay.py b/src/messenger/relay.py index afa5a027..4b6b1f66 100644 --- a/src/messenger/relay.py +++ b/src/messenger/relay.py @@ -3,16 +3,19 @@ import signal from dishka import make_async_container from faststream.rabbit import Channel, RabbitBroker +from faststream.rabbit.opentelemetry import RabbitTelemetryMiddleware from messenger.attachments.config import PipelineSettings from messenger.attachments.entrypoints.heartbeat import Heartbeat from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal from messenger.attachments.entrypoints.relay import Relay +from messenger.attachments.infrastructure.queue_metrics import QueueMetrics from messenger.shared.application.messagebus import EventBus from messenger.shared.config import PlatformSettings from messenger.shared.entrypoints.ioc import PlatformProvider from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger +from messenger.shared.infrastructure.postgres import Database from messenger.shared.infrastructure.telemetry import start_telemetry PING_TIMEOUT_SECONDS = 2.0 @@ -28,6 +31,7 @@ async def serve() -> None: pipeline.broker_url.get_secret_value(), default_channel=Channel(publisher_confirms=True, on_return_raises=True), logger=None, + middlewares=() if telemetry is None else (RabbitTelemetryMiddleware(),), ) container = make_async_container( PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker) @@ -47,6 +51,8 @@ async def serve() -> None: pipeline.heartbeat_interval_seconds, logger, ) + queue_metrics = QueueMetrics(await container.get(Database), logger) + queue_metrics.start() relay.start() heartbeat.start() logger.info("relay.started") @@ -54,6 +60,7 @@ async def serve() -> None: await stop.wait() finally: await heartbeat.stop() + await queue_metrics.stop() await relay.stop() await bus.stop() await broker.stop() diff --git a/src/messenger/shared/infrastructure/postgres.py b/src/messenger/shared/infrastructure/postgres.py index ccbcfd4b..f2c7315e 100644 --- a/src/messenger/shared/infrastructure/postgres.py +++ b/src/messenger/shared/infrastructure/postgres.py @@ -1,13 +1,16 @@ import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from typing import ClassVar, Protocol +from typing import ClassVar, Protocol, cast +from opentelemetry import metrics +from opentelemetry.metrics import Observation from sqlalchemy import Text, make_url, select from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError from sqlalchemy.exc import TimeoutError as PoolTimeout from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.pool import AsyncAdaptedQueuePool, ConnectionPoolEntry from messenger.shared.application.errors import Unavailable from messenger.shared.domain.events import DomainEvent @@ -38,6 +41,17 @@ def unavailable(error: BaseException) -> bool: return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03") +class ObservedPool(AsyncAdaptedQueuePool): + waiting = 0 + + def _do_get(self) -> ConnectionPoolEntry: + self.waiting += 1 + try: + return super()._do_get() + finally: + self.waiting -= 1 + + class Database: __slots__ = ("_engine", "_sessions") @@ -52,6 +66,7 @@ class Database: ) -> None: self._engine = create_async_engine( make_url(dsn).set(drivername="postgresql+asyncpg"), + poolclass=ObservedPool, pool_size=max_size, max_overflow=0, pool_timeout=acquire_timeout, @@ -68,6 +83,22 @@ class Database: }, }, ) + meter = metrics.get_meter("messenger.database") + meter.create_observable_gauge( + "messenger.db.pool.used", + callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).checkedout())]], + unit="{connection}", + ) + meter.create_observable_gauge( + "messenger.db.pool.waiting", + callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).waiting)]], + unit="{request}", + ) + meter.create_observable_gauge( + "messenger.db.pool.capacity", + callbacks=[lambda _: [Observation(max_size)]], + unit="{connection}", + ) self._sessions = async_sessionmaker(self._engine, expire_on_commit=False) async def open(self) -> None: diff --git a/src/messenger/shared/infrastructure/runtime_metrics.py b/src/messenger/shared/infrastructure/runtime_metrics.py new file mode 100644 index 00000000..c4645bb2 --- /dev/null +++ b/src/messenger/shared/infrastructure/runtime_metrics.py @@ -0,0 +1,26 @@ +import os +import resource +from pathlib import Path + +from opentelemetry.metrics import MeterProvider, Observation + + +def rss() -> int: + try: + return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE") + except OSError: + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + + +def observe_process(provider: MeterProvider) -> None: + meter = provider.get_meter("messenger.process") + meter.create_observable_counter( + "process.cpu.time", + unit="s", + callbacks=[lambda _: [Observation(sum(os.times()[:2]))]], + ) + meter.create_observable_gauge( + "process.memory.usage", + unit="By", + callbacks=[lambda _: [Observation(rss())]], + ) diff --git a/src/messenger/shared/infrastructure/telemetry.py b/src/messenger/shared/infrastructure/telemetry.py index e78d3d5e..cd2e9d3b 100644 --- a/src/messenger/shared/infrastructure/telemetry.py +++ b/src/messenger/shared/infrastructure/telemetry.py @@ -25,6 +25,7 @@ from opentelemetry.trace import Link, SpanKind, TraceState from opentelemetry.util.types import Attributes from messenger.shared.config import PlatformSettings +from messenger.shared.infrastructure.runtime_metrics import observe_process SERVICE_VERSION = version("mtusi-messenger") REQUEST_ID = "messenger.request_id" @@ -127,6 +128,7 @@ def start_telemetry( views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())], shutdown_on_exit=False, ) + observe_process(meter) logs = LoggerProvider(resource=resource, shutdown_on_exit=False) logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) trace.set_tracer_provider(tracer) diff --git a/src/messenger/shared/infrastructure/trace_context.py b/src/messenger/shared/infrastructure/trace_context.py new file mode 100644 index 00000000..c53f2082 --- /dev/null +++ b/src/messenger/shared/infrastructure/trace_context.py @@ -0,0 +1,22 @@ +from collections.abc import Iterator +from contextlib import contextmanager + +from opentelemetry.context import attach, detach +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +PROPAGATOR = TraceContextTextMapPropagator() + + +def capture_context() -> dict[str, str]: + carrier: dict[str, str] = {} + PROPAGATOR.inject(carrier) + return carrier + + +@contextmanager +def restored_context(carrier: dict[str, str] | None) -> Iterator[None]: + token = attach(PROPAGATOR.extract(carrier or {})) + try: + yield + finally: + detach(token) diff --git a/student-tests/api/test_bounded_contexts.py b/student-tests/api/test_bounded_contexts.py index 98935113..072648fc 100644 --- a/student-tests/api/test_bounded_contexts.py +++ b/student-tests/api/test_bounded_contexts.py @@ -112,6 +112,8 @@ def test_shared_core_holds_only_platform() -> None: "logging.py", "postgres.py", "telemetry.py", + "trace_context.py", + "runtime_metrics.py", "readiness.py", } diff --git a/student-tests/api/test_migrations.py b/student-tests/api/test_migrations.py index 6172f0d3..0b56cb5f 100644 --- a/student-tests/api/test_migrations.py +++ b/student-tests/api/test_migrations.py @@ -6,6 +6,7 @@ import sys from collections.abc import AsyncIterator from datetime import UTC, datetime, timedelta from pathlib import Path +from typing import Any from uuid import uuid4 import pytest @@ -15,6 +16,7 @@ from alembic.migration import MigrationContext from alembic.script import ScriptDirectory from alembic.util import CommandError from sqlalchemy import MetaData, Table, case, func, insert, inspect, literal, make_url, select, text +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.pool import NullPool @@ -44,6 +46,7 @@ VERSIONS = [ "0004_jobs", "0005_failed_registry", "0006_attachment_quarantine", + "0007_trace_context", ] TABLES = [ UserRow, @@ -247,13 +250,15 @@ async def contents(dsn: str) -> dict: table = await connection.run_sync( lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync) ) - missing = {} + missing: dict[str, Any] = {} if model is AttachmentRow and "published_version" not in table.c: missing = { "original_key": case((table.c.status == "ready", table.c.object_key)), "staging_key": case((table.c.status == "pending", table.c.object_key)), "published_version": case((table.c.status == "ready", 1), else_=0), } + if model in (JobRow, OutboxRow): + missing["trace_context"] = literal({}, type_=JSONB) columns = [ table.c[col.name] if col.name in table.c @@ -449,7 +454,7 @@ async def test_failed_revision_rolls_back_ddl_and_version( "from alembic import op\n" "import sqlalchemy as sa\n" "revision = '0003_failure'\n" - "down_revision = '0006_attachment_quarantine'\n" + "down_revision = '0007_trace_context'\n" "def upgrade():\n" " op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n" " raise RuntimeError('migration failed')\n" diff --git a/student-tests/api/test_trace_context.py b/student-tests/api/test_trace_context.py new file mode 100644 index 00000000..6e2b0058 --- /dev/null +++ b/student-tests/api/test_trace_context.py @@ -0,0 +1,65 @@ +import asyncio +from datetime import timedelta +from uuid import UUID + +from httpx import AsyncClient +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import SpanKind +from sqlalchemy import select +from test_pipeline import accepted, job_of + +from conftest import sql_connection +from messenger.attachments.infrastructure.postgres import JobRow, OutboxRow, PostgresJobRepository +from messenger.shared.infrastructure.trace_context import capture_context, restored_context +from pipeline import pipeline_of + + +async def test_context_survives_outbox_delivery_and_lease_recovery(client: AsyncClient) -> None: + provider = TracerProvider(shutdown_on_exit=False) + tracer = provider.get_tracer("test") + with tracer.start_as_current_span("upload", kind=SpanKind.SERVER): + origin = capture_context() + _, _, attachment_id = await accepted(client) + job = await job_of(attachment_id) + async with sql_connection() as connection: + assert ( + await connection.scalar(select(JobRow.trace_context).where(JobRow.job_id == job.id)) + == origin + ) + assert ( + await connection.scalar( + select(OutboxRow.trace_context).where(OutboxRow.job_id == job.id) + ) + == origin + ) + pipeline = await pipeline_of(client) + await pipeline.publish() + assert pipeline.publisher.contexts == [origin] + uow = pipeline.uow() + async with uow.transaction(): + claim = await uow.jobs.claim(job, UUID(int=1), timedelta(seconds=-1)) + assert claim is not None + await uow.commit() + await pipeline.recover() + await asyncio.sleep(0.02) + await pipeline.publish() + assert len(pipeline.publisher.contexts) == 2 + assert pipeline.publisher.contexts[-1] == origin + with restored_context(origin), tracer.start_as_current_span("retry", kind=SpanKind.CONSUMER): + retry = capture_context() + async with uow.transaction(): + await PostgresJobRepository(uow.session)._enqueue(job) + await uow.commit() + async with sql_connection() as connection: + assert ( + await connection.scalar( + select(OutboxRow.trace_context) + .where(OutboxRow.job_id == job.id) + .order_by(OutboxRow.id.desc()) + .limit(1) + ) + == retry + ) + assert retry["traceparent"].split("-")[1] == origin["traceparent"].split("-")[1] + assert retry != origin + provider.shutdown() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py index 09d8689c..dcbd4df9 100644 --- a/student-tests/pipeline.py +++ b/student-tests/pipeline.py @@ -60,18 +60,22 @@ class InlineRenderer: class CapturingPublisher: - __slots__ = ("bodies", "failing", "job_ids") + __slots__ = ("bodies", "contexts", "failing", "job_ids") def __init__(self) -> None: self.bodies: list[bytes] = [] + self.contexts: list[dict[str, str]] = [] self.job_ids: list[UUID] = [] self.failing = False - async def publish(self, job_id: UUID, body: bytes) -> None: + async def publish( + self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None + ) -> None: if self.failing: raise Unavailable("брокер недоступен") self.job_ids.append(job_id) self.bodies.append(body) + self.contexts.append(trace_context or {}) class InlinePipeline: diff --git a/student-tests/unit/test_job_observer.py b/student-tests/unit/test_job_observer.py index a8cce472..486524e7 100644 --- a/student-tests/unit/test_job_observer.py +++ b/student-tests/unit/test_job_observer.py @@ -31,6 +31,7 @@ def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None: for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")): with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER): observer.observe(item, outcome, 0.3) + observer.active(1) data = reader.get_metrics_data() assert data is not None found = { @@ -39,6 +40,18 @@ def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None: for scope in resource.scope_metrics for metric in scope.metrics } + assert ( + sum( + p.count + for p in found["messenger.job.time_to_ready"].data.data_points + if isinstance(p, HistogramDataPoint) + ) + == 1 + ) + busy = next(iter(found["messenger.worker.busy"].data.data_points)) + assert isinstance(busy, NumberDataPoint) + assert busy.value == 1 + observer.active(-1) counted = { (point.attributes or {})["outcome"]: point.value for point in found["messenger.jobs.processed"].data.data_points