feat: Add live dashboard and freeze SLO alerts (U3/U4 setup)

This commit is contained in:
Самохвалов Андрей Ремович
2026-10-02 15:32:51 +03:00
parent 9fac497d1a
commit 5deecdf545
15 changed files with 976 additions and 22 deletions
+2
View File
@@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
GRAFANA_PORT=3000
PROMETHEUS_PORT=9090
+3
View File
@@ -13,6 +13,8 @@ services:
prometheus:
image: prom/prometheus:v3.5.0
ports:
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
@@ -20,6 +22,7 @@ services:
- --web.enable-remote-write-receiver
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro
- prometheus_data:/prometheus
loki:
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.01,
"concurrency": 4,
"successful_requests": 18260,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 608.462,
"successful_latency_ms": {
"p50": 4.289,
"p95": 14.966,
"p99": 59.664
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,40 @@
{
"accepted_ready": 40,
"concurrency": 4,
"elapsed_seconds": 13.029,
"failed": 0,
"failure_types": [],
"lab": 6,
"per_size": 20,
"profiles": {
"noise_800x600": {
"ready_ms": {
"p50": 1309.41,
"p95": 1618.28,
"p99": 1645.672
},
"size_bytes": 1441103,
"successful": 20,
"upload_ms": {
"p50": 34.166,
"p95": 52.97,
"p99": 84.31
}
},
"small_320x200": {
"ready_ms": {
"p50": 988.508,
"p95": 1304.924,
"p99": 1332.116
},
"size_bytes": 111817,
"successful": 20,
"upload_ms": {
"p50": 24.969,
"p95": 31.176,
"p99": 33.817
}
}
},
"workload": "synthetic PNG upload and poll until ready"
}
+48
View File
@@ -0,0 +1,48 @@
# Зафиксированные SLO до инъекций отказов — 2026-10-02
Стенд: Colima/arm64, 2 CPU, 2 GiB RAM, две API-реплики за HAProxy,
serial worker, PostgreSQL/MinIO/RabbitMQ, Collector/Prometheus/Loki/Tempo/
Grafana. На момент фиксации fault matrix из перенесённой lab5 выполнена, но
**SLO-инциденты ЛР6 ещё не запускались**. Правила ниже лежат в
`infra/prometheus/rules.yml`; `promtool check rules` обнаружил 6 правил.
## Baseline
- 30,010 с, четыре closed-loop клиента, 18 260 успешных POST сообщений,
0 ошибок, 608,462 RPS, latency успехов p50/p95/p99
4,289/14,966/59,664 мс: `lab6-http-baseline-2api-2026-10-02.json`.
- 20 PNG 320×200 (111 817 байт) + 20 PNG 800×600 (1 441 103 байта),
четыре клиента: 40/40 `ready`, 0 ошибок, 13,029 с. p95 от начала upload
до ready 1,305/1,618 с по профилям: `lab6-image-baseline-2api-2026-10-02.json`.
Poll раз в 0,3 с; SDK histogram измеряет от durable job creation до result
COMMIT и не включает upload до принятия.
## Цели и формулы
- **API:** 99% бизнес HTTP-запросов без 5xx за скользящие 10 минут.
Numerator — 5xx, denominator — все non-health ответы, включая 4xx;
HAProxy-generated 503 не видны SDK (ограничение), поэтому эксперимент
обращается к живому API-процессу напрямую при DB outage. При нулевом
traffic SLI неизвестен, а не равен 100%.
- **Ready:** 95% завершённых image jobs достигают `ready` не позднее 5 секунд
после durable acceptance за скользящие 10 минут. Числитель плохих jobs:
`rate(ready_duration_count) - rate(ready_duration_bucket{le="5"})`,
знаменатель — `rate(ready_duration_count)`. Застрявшие jobs ещё не
завершены и не попадают в histogram; queue oldest age и отдельный alert
закрывают слепую зону.
Локальные burn-rate окна уменьшены пропорционально времени демонстрации:
short требует одновременно 15 и 30 секунд, threshold 14,4 бюджета и `for:10s`;
long требует одновременно 60 и 120 секунд, threshold 6 и `for:10s`.
Для API error budget 1% это error ratio >14,4% / >6%; для ready budget 5%
slow ratio >72% / >30%. Оценка правил каждые 5 секунд. Operational alerts:
HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при
`for:10s` и sample freshness <15 с. Порог выбран до сценариев;
эксперимент не меняет его после наблюдений.
Профиль опыта: одинаковые API/worker/DB лимиты и синтетические данные.
Короткий сбой ожидается увидеть short-правилом без long; длинный — обоими.
Записываются UTC timestamps воздействия, pending/firing/inactive из
Prometheus rule API каждые 5 с, detection/recovery, false positives на
спокойном baseline и выбранные SLI/queue values. Эти окна учебные и не
являются production SLO.
+563 -17
View File
@@ -1,19 +1,26 @@
{
"uid": "lab6-telemetry",
"title": "Lab 6 · Telemetry stack",
"tags": ["lab6"],
"title": "Lab 6 · Messenger signals",
"tags": [
"lab6"
],
"timezone": "browser",
"schemaVersion": 40,
"version": 1,
"version": 2,
"refresh": "10s",
"time": { "from": "now-1h", "to": "now" },
"time": {
"from": "now-1h",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Telemetry stores and collector reachable",
"description": "Prometheus self-scrape and internal /metrics endpoints. API and worker panels follow their actual exported metric names.",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"title": "Stores",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
@@ -21,26 +28,565 @@
"legendFormat": "{{job}}"
}
],
"fieldConfig": { "defaults": { "min": 0, "max": 1 }, "overrides": [] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" } },
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
}
},
{
"id": 2,
"type": "timeseries",
"title": "Collector exporter queue size",
"description": "Bounded exporter queues, sourced from the Collector's own Prometheus endpoint.",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"title": "HTTP RPS",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))",
"legendFormat": "series 1"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
}
},
{
"id": 3,
"type": "timeseries",
"title": "HTTP 5xx ratio",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)",
"legendFormat": "series 1"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
}
},
{
"id": 4,
"type": "timeseries",
"title": "HTTP p50/p95/p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))",
"legendFormat": "series 1"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))",
"legendFormat": "series 2"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))",
"legendFormat": "series 3"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
}
},
{
"id": 5,
"type": "timeseries",
"title": "Job outcomes",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by(outcome)(rate(messenger_jobs_processed_total[1m]))",
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
}
},
{
"id": 6,
"type": "timeseries",
"title": "Job p95/p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))",
"legendFormat": "series 1"
},
{
"refId": "B",
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))",
"legendFormat": "series 2"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
}
},
{
"id": 7,
"type": "timeseries",
"title": "Queue depth",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_queue_depth{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_depth{queue_name=\"attachments.process\"})<15))",
"legendFormat": "series 1"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
}
},
{
"id": 8,
"type": "timeseries",
"title": "Oldest queued job",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"})<15))",
"legendFormat": "series 1"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
}
},
{
"id": 9,
"type": "timeseries",
"title": "DB pool used / waiting",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by(service_name)(messenger_db_pool_used)",
"legendFormat": "series 1"
},
{
"refId": "B",
"expr": "sum by(service_name)(messenger_db_pool_waiting)",
"legendFormat": "series 2"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 32
}
},
{
"id": 10,
"type": "timeseries",
"title": "Worker busy / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy)",
"legendFormat": "series 1"
},
{
"refId": "B",
"expr": "sum(messenger_worker_capacity)",
"legendFormat": "series 2"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 32
}
},
{
"id": 11,
"type": "timeseries",
"title": "Time to ready p50/p95/p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))",
"legendFormat": "series 1"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))",
"legendFormat": "series 2"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))",
"legendFormat": "series 3"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 40
}
},
{
"id": 12,
"type": "timeseries",
"title": "Process CPU",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))",
"legendFormat": "series 1"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 40
}
},
{
"id": 13,
"type": "timeseries",
"title": "Process RSS",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by(service_name)(messenger_process_rss_bytes)",
"legendFormat": "series 1"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 48
}
},
{
"id": 14,
"type": "timeseries",
"title": "Collector queue / failures",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "otelcol_exporter_queue_size{job=\"otel-collector\"}",
"legendFormat": "{{exporter}}"
"legendFormat": "series 1"
},
{
"refId": "B",
"expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total[1m]))",
"legendFormat": "series 2"
},
{
"refId": "C",
"expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))",
"legendFormat": "series 3"
},
{
"refId": "D",
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
"legendFormat": "series 4"
}
],
"fieldConfig": { "defaults": { "min": 0 }, "overrides": [] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" } },
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 48
}
},
{
"id": 15,
"type": "timeseries",
"title": "Alerts firing",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertstate=\"firing\",alertname=~\"Lab6.*\"}",
"legendFormat": "{{alertname}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 56
}
}
]
}
+5 -1
View File
@@ -1,5 +1,9 @@
global:
scrape_interval: 15s
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/rules.yml
scrape_configs:
- job_name: prometheus
+91
View File
@@ -0,0 +1,91 @@
groups:
- name: lab6-operational
interval: 5s
rules:
- alert: Lab6HighHttpErrorRate
expr: |
sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s]))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])), 0.001) > 0.10
for: 10s
labels:
severity: warning
annotations:
summary: More than 10% of business HTTP requests return 5xx
runbook: See infra/runbooks/incident.md#http-errors
- alert: Lab6QueueLag
expr: |
(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}
and (time() - timestamp(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) < 15)) > 10
for: 10s
labels:
severity: warning
annotations:
summary: Oldest queued image job is over 10 seconds old
runbook: See infra/runbooks/incident.md#queue-lag
- name: lab6-slo-burn
interval: 5s
rules:
- alert: Lab6ApiBurnShort
expr: |
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[15s])) or vector(0))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[15s])),0.001) > 0.144)
and
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) or vector(0))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001) > 0.144)
for: 10s
labels:
severity: page
slo: api
window: short
annotations:
summary: API error budget burning over 14.4x in both 15s and 30s windows
runbook: See infra/runbooks/incident.md#http-errors
- alert: Lab6ApiBurnLong
expr: |
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[60s])) or vector(0))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[60s])),0.001) > 0.06)
and
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[120s])) or vector(0))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[120s])),0.001) > 0.06)
for: 10s
labels:
severity: warning
slo: api
window: long
annotations:
summary: API error budget burning over 6x in both 60s and 120s windows
runbook: See infra/runbooks/incident.md#http-errors
- alert: Lab6ReadyBurnShort
expr: |
((sum(rate(messenger_attachments_ready_duration_seconds_count[15s]))
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[15s])))
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[15s])),0.001) > 0.72)
and
((sum(rate(messenger_attachments_ready_duration_seconds_count[30s]))
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s])))
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001) > 0.72)
for: 10s
labels:
severity: page
slo: ready
window: short
annotations:
summary: Ready-time error budget burning over 14.4x in both 15s and 30s windows
runbook: See infra/runbooks/incident.md#queue-lag
- alert: Lab6ReadyBurnLong
expr: |
((sum(rate(messenger_attachments_ready_duration_seconds_count[60s]))
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[60s])))
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[60s])),0.001) > 0.30)
and
((sum(rate(messenger_attachments_ready_duration_seconds_count[120s]))
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[120s])))
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[120s])),0.001) > 0.30)
for: 10s
labels:
severity: warning
slo: ready
window: long
annotations:
summary: Ready-time error budget burning over 6x in both 60s and 120s windows
runbook: See infra/runbooks/incident.md#queue-lag
+54
View File
@@ -0,0 +1,54 @@
# Lab 6: local incident runbook
Use Grafana at `http://localhost:3000` (dashboard **Lab 6 · Messenger signals**) or
Prometheus rules at `/alerts` inside the Compose network. All times below are
UTC. The dashboard uses stored Prometheus data; Tempo and Loki are separate
datasources. Keep the last hour visible before diagnosing a recovered incident.
## HTTP errors
`Lab6HighHttpErrorRate` fires when 5xx / all non-health HTTP requests exceeds
10% over 30 seconds for another 10 seconds. Client 4xx are in the denominator
and have a separate panel; they are not server failures. Check HTTP RPS first:
an empty or tiny denominator makes an error ratio unreliable. Compare 5xx
ratio with DB pool used/waiting and `up` for the telemetry stores. In Tempo,
find an `HTTP` server span in the time range and inspect its `db.*` child span
duration/status. Query Loki `{service_name="messenger-api"}` for the same
trace ID. A growing pool wait and failing DB span identify a database incident;
a long server span with fast DB children points to the application path.
Restart PostgreSQL only after confirming its container state and persistent
volume. Confirm `up`, 5xx ratio, and `/health/ready` recover; record first
`pending`, `firing`, and `inactive` timestamps from the rule API.
## Queue lag
`Lab6QueueLag` fires when the oldest durable queued job exceeds 10 seconds for
another 10 seconds. `messenger_queue_depth` counts rows with `status='queued'`
in PostgreSQL, including an outbox row awaiting publication. It is sampled by
the publisher. First compare queue depth, oldest age, `messenger_worker_busy`
and `messenger_worker_capacity`. If busy disappears and capacity is absent,
inspect the worker container; if capacity is present but processing outcome
rates stop, inspect broker connectivity and `job_published` events in Loki.
If jobs fail repeatedly, inspect `messenger_jobs_processed_total` grouped by
outcome and the corresponding consumer trace; do not replay until the cause
is corrected. Restart the worker, then verify queued count/age fall, ready
durations appear, and the alert returns to inactive.
## Follow an upload
Take the `trace_id` from the API `attachment_uploaded` log in Loki. Tempo
should contain the API upload span, publisher `attachments.publish` producer
span, worker `attachments.process` consumer span and its S3/render/DB children.
The worker `job_processed` log has the same trace ID. Job identifiers are
synthetic test identifiers; metric labels contain only route templates,
status classes, job type/outcome and queue name. Never copy request bodies,
tokens, signed object URLs or raw image bytes into evidence.
## Missing telemetry
If a signal is absent, check Collector `otelcol_exporter_queue_size`,
`otelcol_exporter_send_failed_*` and store `up` before concluding the business
service recovered. The SDK uses asynchronous bounded queues and 2-second
export timeouts; Collector queues are finite. Export loss during a backend
outage is expected and must be reported. A missing sample is not zero errors.
Telemetry stores are not part of API readiness.
+1 -1
View File
@@ -30,7 +30,7 @@ def main() -> None:
write("postgres_password", password)
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
if not (ROOT / "minio_root_user").exists():
write("minio_root_user", "lab5admin")
write("minio_root_user", "lab6admin")
write("minio_root_password", secrets.token_urlsafe(32))
write("s3_access_key_id", secrets.token_hex(10))
write("s3_secret_access_key", secrets.token_urlsafe(32))
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Capture local Prometheus alert states and SLI values without credentials."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen
QUERIES = {
"http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)',
"queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})',
"ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)',
"collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)',
}
def fetch(base, path, params=None):
url = base + path + ("?" + urlencode(params) if params else "")
with urlopen(url, timeout=4) as response:
result = json.load(response)
if result["status"] != "success":
raise RuntimeError(result)
return result["data"]
def snapshot(base):
groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"]
alerts = [rule for group in groups for rule in group["rules"]]
row = {
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"alerts": {item["name"]: item["state"]
for item in alerts if item["name"].startswith("Lab6")},
"values": {},
}
for name, query in QUERIES.items():
values = fetch(base, "/api/v1/query", {"query": query})["result"]
row["values"][name] = float(values[0]["value"][1]) if values else None
return row
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prometheus", default="http://127.0.0.1:9090")
parser.add_argument("--duration", type=int, default=60)
parser.add_argument("--interval", type=int, default=5)
parser.add_argument("--output", required=True)
args = parser.parse_args()
if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30:
parser.error("invalid duration or interval")
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
end = time.monotonic() + args.duration
with path.open("w", encoding="utf-8") as out:
while True:
row = snapshot(args.prometheus)
out.write(json.dumps(row, sort_keys=True) + "\n")
out.flush()
if time.monotonic() + args.interval > end:
break
time.sleep(args.interval)
print(f"saved {path}")
if __name__ == "__main__":
main()
+2
View File
@@ -26,6 +26,8 @@ def pool() -> ConnectionPool:
open=True,
)
atexit.register(result.close)
from .telemetry import register_pool_metrics
register_pool_metrics(result)
return result
+29 -1
View File
@@ -8,7 +8,8 @@ from pathlib import Path
import pika
import psycopg
from opentelemetry import trace
from opentelemetry import metrics, trace
from opentelemetry.metrics import Observation
from opentelemetry.trace import SpanKind
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from psycopg_pool import PoolTimeout
@@ -24,6 +25,32 @@ tracer = trace.get_tracer(__name__)
trace_context = TraceContextTextMapPropagator()
def register_queue_metrics():
"""Sample the durable backlog; broker delivery does not erase queued jobs."""
meter = metrics.get_meter("messenger.queue")
def backlog(_):
try:
with pool().connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth,"
"coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS oldest_age "
"FROM attachment_jobs WHERE status='queued'"
).fetchone()
return row
except (psycopg.Error, PoolTimeout, OSError):
return None
def observe(field):
def callback(options):
row = backlog(options)
return [Observation(row[field], {"queue_name": QUEUE})] if row is not None else []
return callback
meter.create_observable_gauge("messenger.queue.depth", callbacks=[observe("depth")])
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[observe("oldest_age")], unit="s")
def publish_batch(channel):
with pool().connection() as conn:
token, rows = claim_outbox(conn, limit=10, lease_seconds=30)
@@ -54,6 +81,7 @@ def publish_batch(channel):
def main():
configure("messenger-publisher")
register_queue_metrics()
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").setLevel(logging.ERROR)
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
+42
View File
@@ -2,10 +2,12 @@
import logging
import os
import time
from functools import lru_cache
from uuid import uuid4
from opentelemetry import metrics, trace
from opentelemetry.metrics import Observation
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
@@ -22,6 +24,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10)
JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120)
READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300)
@lru_cache(maxsize=1)
@@ -53,6 +56,7 @@ def configure(service_name: str) -> bool:
views=[
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
View(instrument_name="messenger.jobs.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.attachments.ready_duration", aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
],
))
@@ -65,6 +69,15 @@ def configure(service_name: str) -> bool:
app_logger = logging.getLogger("src")
app_logger.setLevel(logging.INFO)
app_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs))
meter = metrics.get_meter("messenger.runtime")
meter.create_observable_counter("messenger.process.cpu.time", callbacks=[
lambda _: [Observation(time.process_time())],
], unit="s")
def rss(_):
with open("/proc/self/statm", encoding="ascii") as statm:
return [Observation(int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))]
meter.create_observable_gauge("messenger.process.rss", callbacks=[rss], unit="By")
return True
@@ -84,3 +97,32 @@ def job_instruments():
meter.create_counter("messenger.jobs.processed", unit="1"),
meter.create_histogram("messenger.jobs.duration", unit="s"),
)
@lru_cache(maxsize=1)
def pipeline_instruments():
meter = metrics.get_meter("messenger.pipeline")
return (
meter.create_histogram("messenger.attachments.ready_duration", unit="s"),
meter.create_up_down_counter("messenger.worker.busy"),
)
def register_worker_capacity():
metrics.get_meter("messenger.pipeline").create_observable_gauge(
"messenger.worker.capacity", callbacks=[lambda _: [Observation(1)]],
)
def register_pool_metrics(pool):
"""Read local pool counters without querying the database."""
meter = metrics.get_meter("messenger.pool")
def observe(_):
stats = pool.get_stats()
return [Observation(stats["pool_size"] - stats["pool_available"])]
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[observe])
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[
lambda _: [Observation(pool.get_stats()["requests_waiting"])],
])
+10 -2
View File
@@ -9,7 +9,7 @@ import re
import signal
import time
from contextlib import contextmanager
from datetime import datetime
from datetime import datetime, timezone
from io import BytesIO
from pathlib import Path
from uuid import UUID
@@ -26,7 +26,7 @@ from .attachments import BUCKET, storage
from .db import pool
from .image_variants import InvalidImage, MAX_BYTES, render
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
from .telemetry import configure, job_instruments
from .telemetry import configure, job_instruments, pipeline_instruments, register_worker_capacity
logger = logging.getLogger("src.worker")
@@ -151,6 +151,10 @@ def process_message(item: dict) -> str:
with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False):
current = complete_job(conn, job["id"], token, stored)
_fault("after_result_commit_before_ack", job["id"])
if current:
ready_duration, _ = pipeline_instruments()
ready_duration.record(max(0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
{"job_type": "attachment.process.v1"})
if current and job["staging_key"]:
try:
storage().remove_object(BUCKET, job["staging_key"])
@@ -194,6 +198,8 @@ def traced_process_message(item: dict) -> str:
span.set_attribute("job.id", item["job_id"])
span.set_attribute("attachment.id", item["attachment_id"])
started = time.monotonic()
_, busy = pipeline_instruments()
busy.add(1)
outcome = "error"
try:
outcome = process_message(item)
@@ -202,6 +208,7 @@ def traced_process_message(item: dict) -> str:
outcome = "database_unavailable"
raise
finally:
busy.add(-1)
attributes = {"job_type": item["type"], "outcome": outcome}
processed, duration = job_instruments()
processed.add(1, attributes)
@@ -215,6 +222,7 @@ def traced_process_message(item: dict) -> str:
def main() -> None:
configure("messenger-worker")
register_worker_capacity()
logging.basicConfig(level=logging.INFO)
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
raise SystemExit("invalid worker retry settings")