From 5deecdf545f66a94cd11585bc7a39a8ae2986736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 15:32:51 +0300 Subject: [PATCH] feat: Add live dashboard and freeze SLO alerts (U3/U4 setup) --- .env.example | 2 + compose.telemetry.yaml | 3 + .../lab6-http-baseline-2api-2026-10-02.json | 17 + .../lab6-image-baseline-2api-2026-10-02.json | 40 ++ evidence/slo-thresholds-2026-10-02.md | 48 ++ infra/grafana/dashboards/telemetry.json | 580 +++++++++++++++++- infra/prometheus/prometheus.yml | 6 +- infra/prometheus/rules.yml | 91 +++ infra/runbooks/incident.md | 54 ++ scripts/setup-local-secrets.py | 2 +- scripts/slo_watch.py | 69 +++ src/db.py | 2 + src/publisher.py | 30 +- src/telemetry.py | 42 ++ src/worker.py | 12 +- 15 files changed, 976 insertions(+), 22 deletions(-) create mode 100644 evidence/lab6-http-baseline-2api-2026-10-02.json create mode 100644 evidence/lab6-image-baseline-2api-2026-10-02.json create mode 100644 evidence/slo-thresholds-2026-10-02.md create mode 100644 infra/prometheus/rules.yml create mode 100644 infra/runbooks/incident.md create mode 100644 scripts/slo_watch.py diff --git a/.env.example b/.env.example index 2b32655..fbd7637 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9090 diff --git a/compose.telemetry.yaml b/compose.telemetry.yaml index 02f35c4..89358ff 100644 --- a/compose.telemetry.yaml +++ b/compose.telemetry.yaml @@ -13,6 +13,8 @@ services: prometheus: image: prom/prometheus:v3.5.0 + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus @@ -20,6 +22,7 @@ services: - --web.enable-remote-write-receiver volumes: - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro - prometheus_data:/prometheus loki: diff --git a/evidence/lab6-http-baseline-2api-2026-10-02.json b/evidence/lab6-http-baseline-2api-2026-10-02.json new file mode 100644 index 0000000..0751892 --- /dev/null +++ b/evidence/lab6-http-baseline-2api-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.01, + "concurrency": 4, + "successful_requests": 18260, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 608.462, + "successful_latency_ms": { + "p50": 4.289, + "p95": 14.966, + "p99": 59.664 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-image-baseline-2api-2026-10-02.json b/evidence/lab6-image-baseline-2api-2026-10-02.json new file mode 100644 index 0000000..408b1c7 --- /dev/null +++ b/evidence/lab6-image-baseline-2api-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 13.029, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 1309.41, + "p95": 1618.28, + "p99": 1645.672 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 34.166, + "p95": 52.97, + "p99": 84.31 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 988.508, + "p95": 1304.924, + "p99": 1332.116 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 24.969, + "p95": 31.176, + "p99": 33.817 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/slo-thresholds-2026-10-02.md b/evidence/slo-thresholds-2026-10-02.md new file mode 100644 index 0000000..4c39d7d --- /dev/null +++ b/evidence/slo-thresholds-2026-10-02.md @@ -0,0 +1,48 @@ +# Зафиксированные SLO до инъекций отказов — 2026-10-02 + +Стенд: Colima/arm64, 2 CPU, 2 GiB RAM, две API-реплики за HAProxy, +serial worker, PostgreSQL/MinIO/RabbitMQ, Collector/Prometheus/Loki/Tempo/ +Grafana. На момент фиксации fault matrix из перенесённой lab5 выполнена, но +**SLO-инциденты ЛР6 ещё не запускались**. Правила ниже лежат в +`infra/prometheus/rules.yml`; `promtool check rules` обнаружил 6 правил. + +## Baseline + +- 30,010 с, четыре closed-loop клиента, 18 260 успешных POST сообщений, + 0 ошибок, 608,462 RPS, latency успехов p50/p95/p99 + 4,289/14,966/59,664 мс: `lab6-http-baseline-2api-2026-10-02.json`. +- 20 PNG 320×200 (111 817 байт) + 20 PNG 800×600 (1 441 103 байта), + четыре клиента: 40/40 `ready`, 0 ошибок, 13,029 с. p95 от начала upload + до ready 1,305/1,618 с по профилям: `lab6-image-baseline-2api-2026-10-02.json`. + Poll раз в 0,3 с; SDK histogram измеряет от durable job creation до result + COMMIT и не включает upload до принятия. + +## Цели и формулы + +- **API:** 99% бизнес HTTP-запросов без 5xx за скользящие 10 минут. + Numerator — 5xx, denominator — все non-health ответы, включая 4xx; + HAProxy-generated 503 не видны SDK (ограничение), поэтому эксперимент + обращается к живому API-процессу напрямую при DB outage. При нулевом + traffic SLI неизвестен, а не равен 100%. +- **Ready:** 95% завершённых image jobs достигают `ready` не позднее 5 секунд + после durable acceptance за скользящие 10 минут. Числитель плохих jobs: + `rate(ready_duration_count) - rate(ready_duration_bucket{le="5"})`, + знаменатель — `rate(ready_duration_count)`. Застрявшие jobs ещё не + завершены и не попадают в histogram; queue oldest age и отдельный alert + закрывают слепую зону. + +Локальные burn-rate окна уменьшены пропорционально времени демонстрации: +short требует одновременно 15 и 30 секунд, threshold 14,4 бюджета и `for:10s`; +long требует одновременно 60 и 120 секунд, threshold 6 и `for:10s`. +Для API error budget 1% это error ratio >14,4% / >6%; для ready budget 5% +slow ratio >72% / >30%. Оценка правил каждые 5 секунд. Operational alerts: +HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при +`for:10s` и sample freshness <15 с. Порог выбран до сценариев; +эксперимент не меняет его после наблюдений. + +Профиль опыта: одинаковые API/worker/DB лимиты и синтетические данные. +Короткий сбой ожидается увидеть short-правилом без long; длинный — обоими. +Записываются UTC timestamps воздействия, pending/firing/inactive из +Prometheus rule API каждые 5 с, detection/recovery, false positives на +спокойном baseline и выбранные SLI/queue values. Эти окна учебные и не +являются production SLO. diff --git a/infra/grafana/dashboards/telemetry.json b/infra/grafana/dashboards/telemetry.json index 4811866..adfc8da 100644 --- a/infra/grafana/dashboards/telemetry.json +++ b/infra/grafana/dashboards/telemetry.json @@ -1,19 +1,26 @@ { "uid": "lab6-telemetry", - "title": "Lab 6 · Telemetry stack", - "tags": ["lab6"], + "title": "Lab 6 · Messenger signals", + "tags": [ + "lab6" + ], "timezone": "browser", "schemaVersion": 40, - "version": 1, + "version": 2, "refresh": "10s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "panels": [ { "id": 1, "type": "timeseries", - "title": "Telemetry stores and collector reachable", - "description": "Prometheus self-scrape and internal /metrics endpoints. API and worker panels follow their actual exported metric names.", - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "title": "Stores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { "refId": "A", @@ -21,26 +28,565 @@ "legendFormat": "{{job}}" } ], - "fieldConfig": { "defaults": { "min": 0, "max": 1 }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 } + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "timeseries", - "title": "Collector exporter queue size", - "description": "Bounded exporter queues, sourced from the Collector's own Prometheus endpoint.", - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "title": "HTTP RPS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "HTTP 5xx ratio", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "HTTP p50/p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 3" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Job outcomes", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(outcome)(rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Job p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_queue_depth{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_depth{queue_name=\"attachments.process\"})<15))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Oldest queued job", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"})<15))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + } + }, + { + "id": 9, + "type": "timeseries", + "title": "DB pool used / waiting", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(messenger_db_pool_used)", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum by(service_name)(messenger_db_pool_waiting)", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Worker busy / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy)", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Time to ready p50/p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 3" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Process CPU", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Process RSS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(messenger_process_rss_bytes)", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Collector queue / failures", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { "refId": "A", "expr": "otelcol_exporter_queue_size{job=\"otel-collector\"}", - "legendFormat": "{{exporter}}" + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total[1m]))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))", + "legendFormat": "series 3" + }, + { + "refId": "D", + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "series 4" } ], - "fieldConfig": { "defaults": { "min": 0 }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 } + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Alerts firing", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\",alertname=~\"Lab6.*\"}", + "legendFormat": "{{alertname}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + } } ] } diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml index b56d200..01012e9 100644 --- a/infra/prometheus/prometheus.yml +++ b/infra/prometheus/prometheus.yml @@ -1,5 +1,9 @@ global: - scrape_interval: 15s + scrape_interval: 5s + evaluation_interval: 5s + +rule_files: + - /etc/prometheus/rules.yml scrape_configs: - job_name: prometheus diff --git a/infra/prometheus/rules.yml b/infra/prometheus/rules.yml new file mode 100644 index 0000000..3cd1414 --- /dev/null +++ b/infra/prometheus/rules.yml @@ -0,0 +1,91 @@ +groups: + - name: lab6-operational + interval: 5s + rules: + - alert: Lab6HighHttpErrorRate + expr: | + sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])), 0.001) > 0.10 + for: 10s + labels: + severity: warning + annotations: + summary: More than 10% of business HTTP requests return 5xx + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6QueueLag + expr: | + (messenger_queue_oldest_age_seconds{queue_name="attachments.process"} + and (time() - timestamp(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) < 15)) > 10 + for: 10s + labels: + severity: warning + annotations: + summary: Oldest queued image job is over 10 seconds old + runbook: See infra/runbooks/incident.md#queue-lag + - name: lab6-slo-burn + interval: 5s + rules: + - alert: Lab6ApiBurnShort + expr: | + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[15s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[15s])),0.001) > 0.144) + and + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001) > 0.144) + for: 10s + labels: + severity: page + slo: api + window: short + annotations: + summary: API error budget burning over 14.4x in both 15s and 30s windows + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6ApiBurnLong + expr: | + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[60s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[60s])),0.001) > 0.06) + and + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[120s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[120s])),0.001) > 0.06) + for: 10s + labels: + severity: warning + slo: api + window: long + annotations: + summary: API error budget burning over 6x in both 60s and 120s windows + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6ReadyBurnShort + expr: | + ((sum(rate(messenger_attachments_ready_duration_seconds_count[15s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[15s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[15s])),0.001) > 0.72) + and + ((sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001) > 0.72) + for: 10s + labels: + severity: page + slo: ready + window: short + annotations: + summary: Ready-time error budget burning over 14.4x in both 15s and 30s windows + runbook: See infra/runbooks/incident.md#queue-lag + - alert: Lab6ReadyBurnLong + expr: | + ((sum(rate(messenger_attachments_ready_duration_seconds_count[60s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[60s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[60s])),0.001) > 0.30) + and + ((sum(rate(messenger_attachments_ready_duration_seconds_count[120s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[120s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[120s])),0.001) > 0.30) + for: 10s + labels: + severity: warning + slo: ready + window: long + annotations: + summary: Ready-time error budget burning over 6x in both 60s and 120s windows + runbook: See infra/runbooks/incident.md#queue-lag diff --git a/infra/runbooks/incident.md b/infra/runbooks/incident.md new file mode 100644 index 0000000..5de27a4 --- /dev/null +++ b/infra/runbooks/incident.md @@ -0,0 +1,54 @@ +# Lab 6: local incident runbook + +Use Grafana at `http://localhost:3000` (dashboard **Lab 6 · Messenger signals**) or +Prometheus rules at `/alerts` inside the Compose network. All times below are +UTC. The dashboard uses stored Prometheus data; Tempo and Loki are separate +datasources. Keep the last hour visible before diagnosing a recovered incident. + +## HTTP errors + +`Lab6HighHttpErrorRate` fires when 5xx / all non-health HTTP requests exceeds +10% over 30 seconds for another 10 seconds. Client 4xx are in the denominator +and have a separate panel; they are not server failures. Check HTTP RPS first: +an empty or tiny denominator makes an error ratio unreliable. Compare 5xx +ratio with DB pool used/waiting and `up` for the telemetry stores. In Tempo, +find an `HTTP` server span in the time range and inspect its `db.*` child span +duration/status. Query Loki `{service_name="messenger-api"}` for the same +trace ID. A growing pool wait and failing DB span identify a database incident; +a long server span with fast DB children points to the application path. +Restart PostgreSQL only after confirming its container state and persistent +volume. Confirm `up`, 5xx ratio, and `/health/ready` recover; record first +`pending`, `firing`, and `inactive` timestamps from the rule API. + +## Queue lag + +`Lab6QueueLag` fires when the oldest durable queued job exceeds 10 seconds for +another 10 seconds. `messenger_queue_depth` counts rows with `status='queued'` +in PostgreSQL, including an outbox row awaiting publication. It is sampled by +the publisher. First compare queue depth, oldest age, `messenger_worker_busy` +and `messenger_worker_capacity`. If busy disappears and capacity is absent, +inspect the worker container; if capacity is present but processing outcome +rates stop, inspect broker connectivity and `job_published` events in Loki. +If jobs fail repeatedly, inspect `messenger_jobs_processed_total` grouped by +outcome and the corresponding consumer trace; do not replay until the cause +is corrected. Restart the worker, then verify queued count/age fall, ready +durations appear, and the alert returns to inactive. + +## Follow an upload + +Take the `trace_id` from the API `attachment_uploaded` log in Loki. Tempo +should contain the API upload span, publisher `attachments.publish` producer +span, worker `attachments.process` consumer span and its S3/render/DB children. +The worker `job_processed` log has the same trace ID. Job identifiers are +synthetic test identifiers; metric labels contain only route templates, +status classes, job type/outcome and queue name. Never copy request bodies, +tokens, signed object URLs or raw image bytes into evidence. + +## Missing telemetry + +If a signal is absent, check Collector `otelcol_exporter_queue_size`, +`otelcol_exporter_send_failed_*` and store `up` before concluding the business +service recovered. The SDK uses asynchronous bounded queues and 2-second +export timeouts; Collector queues are finite. Export loss during a backend +outage is expected and must be reported. A missing sample is not zero errors. +Telemetry stores are not part of API readiness. diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py index 6fbac6d..8cb5829 100644 --- a/scripts/setup-local-secrets.py +++ b/scripts/setup-local-secrets.py @@ -30,7 +30,7 @@ def main() -> None: write("postgres_password", password) write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") if not (ROOT / "minio_root_user").exists(): - write("minio_root_user", "lab5admin") + write("minio_root_user", "lab6admin") write("minio_root_password", secrets.token_urlsafe(32)) write("s3_access_key_id", secrets.token_hex(10)) write("s3_secret_access_key", secrets.token_urlsafe(32)) diff --git a/scripts/slo_watch.py b/scripts/slo_watch.py new file mode 100644 index 0000000..2c0b0eb --- /dev/null +++ b/scripts/slo_watch.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Capture local Prometheus alert states and SLI values without credentials.""" + +import argparse +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen + + +QUERIES = { + "http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)', + "queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})', + "ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)', + "collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)', +} + + +def fetch(base, path, params=None): + url = base + path + ("?" + urlencode(params) if params else "") + with urlopen(url, timeout=4) as response: + result = json.load(response) + if result["status"] != "success": + raise RuntimeError(result) + return result["data"] + + +def snapshot(base): + groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"] + alerts = [rule for group in groups for rule in group["rules"]] + row = { + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "alerts": {item["name"]: item["state"] + for item in alerts if item["name"].startswith("Lab6")}, + "values": {}, + } + for name, query in QUERIES.items(): + values = fetch(base, "/api/v1/query", {"query": query})["result"] + row["values"][name] = float(values[0]["value"][1]) if values else None + return row + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prometheus", default="http://127.0.0.1:9090") + parser.add_argument("--duration", type=int, default=60) + parser.add_argument("--interval", type=int, default=5) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30: + parser.error("invalid duration or interval") + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + end = time.monotonic() + args.duration + with path.open("w", encoding="utf-8") as out: + while True: + row = snapshot(args.prometheus) + out.write(json.dumps(row, sort_keys=True) + "\n") + out.flush() + if time.monotonic() + args.interval > end: + break + time.sleep(args.interval) + print(f"saved {path}") + + +if __name__ == "__main__": + main() diff --git a/src/db.py b/src/db.py index 832ff07..48947b0 100644 --- a/src/db.py +++ b/src/db.py @@ -26,6 +26,8 @@ def pool() -> ConnectionPool: open=True, ) atexit.register(result.close) + from .telemetry import register_pool_metrics + register_pool_metrics(result) return result diff --git a/src/publisher.py b/src/publisher.py index 209d24b..02d2590 100644 --- a/src/publisher.py +++ b/src/publisher.py @@ -8,7 +8,8 @@ from pathlib import Path import pika import psycopg -from opentelemetry import trace +from opentelemetry import metrics, trace +from opentelemetry.metrics import Observation from opentelemetry.trace import SpanKind from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from psycopg_pool import PoolTimeout @@ -24,6 +25,32 @@ tracer = trace.get_tracer(__name__) trace_context = TraceContextTextMapPropagator() +def register_queue_metrics(): + """Sample the durable backlog; broker delivery does not erase queued jobs.""" + meter = metrics.get_meter("messenger.queue") + + def backlog(_): + try: + with pool().connection() as conn: + row = conn.execute( + "SELECT count(*) AS depth," + "coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS oldest_age " + "FROM attachment_jobs WHERE status='queued'" + ).fetchone() + return row + except (psycopg.Error, PoolTimeout, OSError): + return None + + def observe(field): + def callback(options): + row = backlog(options) + return [Observation(row[field], {"queue_name": QUEUE})] if row is not None else [] + return callback + + meter.create_observable_gauge("messenger.queue.depth", callbacks=[observe("depth")]) + meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[observe("oldest_age")], unit="s") + + def publish_batch(channel): with pool().connection() as conn: token, rows = claim_outbox(conn, limit=10, lease_seconds=30) @@ -54,6 +81,7 @@ def publish_batch(channel): def main(): configure("messenger-publisher") + register_queue_metrics() logging.basicConfig(level=logging.INFO) logging.getLogger("pika").setLevel(logging.ERROR) url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() diff --git a/src/telemetry.py b/src/telemetry.py index 9d6408c..8d68d31 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -2,10 +2,12 @@ import logging import os +import time from functools import lru_cache from uuid import uuid4 from opentelemetry import metrics, trace +from opentelemetry.metrics import Observation from opentelemetry._logs import set_logger_provider from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter @@ -22,6 +24,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10) JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120) +READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300) @lru_cache(maxsize=1) @@ -53,6 +56,7 @@ def configure(service_name: str) -> bool: views=[ View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), View(instrument_name="messenger.jobs.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachments.ready_duration", aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), ], )) @@ -65,6 +69,15 @@ def configure(service_name: str) -> bool: app_logger = logging.getLogger("src") app_logger.setLevel(logging.INFO) app_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + meter = metrics.get_meter("messenger.runtime") + meter.create_observable_counter("messenger.process.cpu.time", callbacks=[ + lambda _: [Observation(time.process_time())], + ], unit="s") + def rss(_): + with open("/proc/self/statm", encoding="ascii") as statm: + return [Observation(int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))] + + meter.create_observable_gauge("messenger.process.rss", callbacks=[rss], unit="By") return True @@ -84,3 +97,32 @@ def job_instruments(): meter.create_counter("messenger.jobs.processed", unit="1"), meter.create_histogram("messenger.jobs.duration", unit="s"), ) + + +@lru_cache(maxsize=1) +def pipeline_instruments(): + meter = metrics.get_meter("messenger.pipeline") + return ( + meter.create_histogram("messenger.attachments.ready_duration", unit="s"), + meter.create_up_down_counter("messenger.worker.busy"), + ) + + +def register_worker_capacity(): + metrics.get_meter("messenger.pipeline").create_observable_gauge( + "messenger.worker.capacity", callbacks=[lambda _: [Observation(1)]], + ) + + +def register_pool_metrics(pool): + """Read local pool counters without querying the database.""" + meter = metrics.get_meter("messenger.pool") + + def observe(_): + stats = pool.get_stats() + return [Observation(stats["pool_size"] - stats["pool_available"])] + + meter.create_observable_gauge("messenger.db.pool.used", callbacks=[observe]) + meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[ + lambda _: [Observation(pool.get_stats()["requests_waiting"])], + ]) diff --git a/src/worker.py b/src/worker.py index ddcede4..b51e8a0 100644 --- a/src/worker.py +++ b/src/worker.py @@ -9,7 +9,7 @@ import re import signal import time from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timezone from io import BytesIO from pathlib import Path from uuid import UUID @@ -26,7 +26,7 @@ from .attachments import BUCKET, storage from .db import pool from .image_variants import InvalidImage, MAX_BYTES, render from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired -from .telemetry import configure, job_instruments +from .telemetry import configure, job_instruments, pipeline_instruments, register_worker_capacity logger = logging.getLogger("src.worker") @@ -151,6 +151,10 @@ def process_message(item: dict) -> str: with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False): current = complete_job(conn, job["id"], token, stored) _fault("after_result_commit_before_ack", job["id"]) + if current: + ready_duration, _ = pipeline_instruments() + ready_duration.record(max(0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()), + {"job_type": "attachment.process.v1"}) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) @@ -194,6 +198,8 @@ def traced_process_message(item: dict) -> str: span.set_attribute("job.id", item["job_id"]) span.set_attribute("attachment.id", item["attachment_id"]) started = time.monotonic() + _, busy = pipeline_instruments() + busy.add(1) outcome = "error" try: outcome = process_message(item) @@ -202,6 +208,7 @@ def traced_process_message(item: dict) -> str: outcome = "database_unavailable" raise finally: + busy.add(-1) attributes = {"job_type": item["type"], "outcome": outcome} processed, duration = job_instruments() processed.add(1, attributes) @@ -215,6 +222,7 @@ def traced_process_message(item: dict) -> str: def main() -> None: configure("messenger-worker") + register_worker_capacity() logging.basicConfig(level=logging.INFO) if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: raise SystemExit("invalid worker retry settings")