feat: Add live dashboard and freeze SLO alerts (U3/U4 setup)
This commit is contained in:
@@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
GRAFANA_PORT=3000
|
||||
PROMETHEUS_PORT=9090
|
||||
|
||||
@@ -13,6 +13,8 @@ services:
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.5.0
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
@@ -20,6 +22,7 @@ services:
|
||||
- --web.enable-remote-write-receiver
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
|
||||
loki:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.01,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 18260,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 608.462,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.289,
|
||||
"p95": 14.966,
|
||||
"p99": 59.664
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"accepted_ready": 40,
|
||||
"concurrency": 4,
|
||||
"elapsed_seconds": 13.029,
|
||||
"failed": 0,
|
||||
"failure_types": [],
|
||||
"lab": 6,
|
||||
"per_size": 20,
|
||||
"profiles": {
|
||||
"noise_800x600": {
|
||||
"ready_ms": {
|
||||
"p50": 1309.41,
|
||||
"p95": 1618.28,
|
||||
"p99": 1645.672
|
||||
},
|
||||
"size_bytes": 1441103,
|
||||
"successful": 20,
|
||||
"upload_ms": {
|
||||
"p50": 34.166,
|
||||
"p95": 52.97,
|
||||
"p99": 84.31
|
||||
}
|
||||
},
|
||||
"small_320x200": {
|
||||
"ready_ms": {
|
||||
"p50": 988.508,
|
||||
"p95": 1304.924,
|
||||
"p99": 1332.116
|
||||
},
|
||||
"size_bytes": 111817,
|
||||
"successful": 20,
|
||||
"upload_ms": {
|
||||
"p50": 24.969,
|
||||
"p95": 31.176,
|
||||
"p99": 33.817
|
||||
}
|
||||
}
|
||||
},
|
||||
"workload": "synthetic PNG upload and poll until ready"
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
# Зафиксированные SLO до инъекций отказов — 2026-10-02
|
||||
|
||||
Стенд: Colima/arm64, 2 CPU, 2 GiB RAM, две API-реплики за HAProxy,
|
||||
serial worker, PostgreSQL/MinIO/RabbitMQ, Collector/Prometheus/Loki/Tempo/
|
||||
Grafana. На момент фиксации fault matrix из перенесённой lab5 выполнена, но
|
||||
**SLO-инциденты ЛР6 ещё не запускались**. Правила ниже лежат в
|
||||
`infra/prometheus/rules.yml`; `promtool check rules` обнаружил 6 правил.
|
||||
|
||||
## Baseline
|
||||
|
||||
- 30,010 с, четыре closed-loop клиента, 18 260 успешных POST сообщений,
|
||||
0 ошибок, 608,462 RPS, latency успехов p50/p95/p99
|
||||
4,289/14,966/59,664 мс: `lab6-http-baseline-2api-2026-10-02.json`.
|
||||
- 20 PNG 320×200 (111 817 байт) + 20 PNG 800×600 (1 441 103 байта),
|
||||
четыре клиента: 40/40 `ready`, 0 ошибок, 13,029 с. p95 от начала upload
|
||||
до ready 1,305/1,618 с по профилям: `lab6-image-baseline-2api-2026-10-02.json`.
|
||||
Poll раз в 0,3 с; SDK histogram измеряет от durable job creation до result
|
||||
COMMIT и не включает upload до принятия.
|
||||
|
||||
## Цели и формулы
|
||||
|
||||
- **API:** 99% бизнес HTTP-запросов без 5xx за скользящие 10 минут.
|
||||
Numerator — 5xx, denominator — все non-health ответы, включая 4xx;
|
||||
HAProxy-generated 503 не видны SDK (ограничение), поэтому эксперимент
|
||||
обращается к живому API-процессу напрямую при DB outage. При нулевом
|
||||
traffic SLI неизвестен, а не равен 100%.
|
||||
- **Ready:** 95% завершённых image jobs достигают `ready` не позднее 5 секунд
|
||||
после durable acceptance за скользящие 10 минут. Числитель плохих jobs:
|
||||
`rate(ready_duration_count) - rate(ready_duration_bucket{le="5"})`,
|
||||
знаменатель — `rate(ready_duration_count)`. Застрявшие jobs ещё не
|
||||
завершены и не попадают в histogram; queue oldest age и отдельный alert
|
||||
закрывают слепую зону.
|
||||
|
||||
Локальные burn-rate окна уменьшены пропорционально времени демонстрации:
|
||||
short требует одновременно 15 и 30 секунд, threshold 14,4 бюджета и `for:10s`;
|
||||
long требует одновременно 60 и 120 секунд, threshold 6 и `for:10s`.
|
||||
Для API error budget 1% это error ratio >14,4% / >6%; для ready budget 5%
|
||||
slow ratio >72% / >30%. Оценка правил каждые 5 секунд. Operational alerts:
|
||||
HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при
|
||||
`for:10s` и sample freshness <15 с. Порог выбран до сценариев;
|
||||
эксперимент не меняет его после наблюдений.
|
||||
|
||||
Профиль опыта: одинаковые API/worker/DB лимиты и синтетические данные.
|
||||
Короткий сбой ожидается увидеть short-правилом без long; длинный — обоими.
|
||||
Записываются UTC timestamps воздействия, pending/firing/inactive из
|
||||
Prometheus rule API каждые 5 с, detection/recovery, false positives на
|
||||
спокойном baseline и выбранные SLI/queue values. Эти окна учебные и не
|
||||
являются production SLO.
|
||||
@@ -1,19 +1,26 @@
|
||||
{
|
||||
"uid": "lab6-telemetry",
|
||||
"title": "Lab 6 · Telemetry stack",
|
||||
"tags": ["lab6"],
|
||||
"title": "Lab 6 · Messenger signals",
|
||||
"tags": [
|
||||
"lab6"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 40,
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "Telemetry stores and collector reachable",
|
||||
"description": "Prometheus self-scrape and internal /metrics endpoints. API and worker panels follow their actual exported metric names.",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"title": "Stores",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
@@ -21,26 +28,565 @@
|
||||
"legendFormat": "{{job}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "min": 0, "max": 1 }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom" } },
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Collector exporter queue size",
|
||||
"description": "Bounded exporter queues, sourced from the Collector's own Prometheus endpoint.",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"title": "HTTP RPS",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))",
|
||||
"legendFormat": "series 1"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP 5xx ratio",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)",
|
||||
"legendFormat": "series 1"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p50/p95/p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))",
|
||||
"legendFormat": "series 1"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))",
|
||||
"legendFormat": "series 2"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))",
|
||||
"legendFormat": "series 3"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Job outcomes",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(outcome)(rate(messenger_jobs_processed_total[1m]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Job p95/p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "series 1"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "series 2"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_queue_depth{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_depth{queue_name=\"attachments.process\"})<15))",
|
||||
"legendFormat": "series 1"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Oldest queued job",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"})<15))",
|
||||
"legendFormat": "series 1"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 24
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool used / waiting",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name)(messenger_db_pool_used)",
|
||||
"legendFormat": "series 1"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by(service_name)(messenger_db_pool_waiting)",
|
||||
"legendFormat": "series 2"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Worker busy / capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy)",
|
||||
"legendFormat": "series 1"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(messenger_worker_capacity)",
|
||||
"legendFormat": "series 2"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Time to ready p50/p95/p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "series 1"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "series 2"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "series 3"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 40
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))",
|
||||
"legendFormat": "series 1"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 40
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Process RSS",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name)(messenger_process_rss_bytes)",
|
||||
"legendFormat": "series 1"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Collector queue / failures",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "otelcol_exporter_queue_size{job=\"otel-collector\"}",
|
||||
"legendFormat": "{{exporter}}"
|
||||
"legendFormat": "series 1"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total[1m]))",
|
||||
"legendFormat": "series 2"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))",
|
||||
"legendFormat": "series 3"
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
|
||||
"legendFormat": "series 4"
|
||||
}
|
||||
],
|
||||
"fieldConfig": { "defaults": { "min": 0 }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom" } },
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "timeseries",
|
||||
"title": "Alerts firing",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertstate=\"firing\",alertname=~\"Lab6.*\"}",
|
||||
"legendFormat": "{{alertname}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 56
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
groups:
|
||||
- name: lab6-operational
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: Lab6HighHttpErrorRate
|
||||
expr: |
|
||||
sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])), 0.001) > 0.10
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: More than 10% of business HTTP requests return 5xx
|
||||
runbook: See infra/runbooks/incident.md#http-errors
|
||||
- alert: Lab6QueueLag
|
||||
expr: |
|
||||
(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}
|
||||
and (time() - timestamp(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) < 15)) > 10
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest queued image job is over 10 seconds old
|
||||
runbook: See infra/runbooks/incident.md#queue-lag
|
||||
- name: lab6-slo-burn
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: Lab6ApiBurnShort
|
||||
expr: |
|
||||
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[15s])) or vector(0))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[15s])),0.001) > 0.144)
|
||||
and
|
||||
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) or vector(0))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001) > 0.144)
|
||||
for: 10s
|
||||
labels:
|
||||
severity: page
|
||||
slo: api
|
||||
window: short
|
||||
annotations:
|
||||
summary: API error budget burning over 14.4x in both 15s and 30s windows
|
||||
runbook: See infra/runbooks/incident.md#http-errors
|
||||
- alert: Lab6ApiBurnLong
|
||||
expr: |
|
||||
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[60s])) or vector(0))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[60s])),0.001) > 0.06)
|
||||
and
|
||||
((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[120s])) or vector(0))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[120s])),0.001) > 0.06)
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
slo: api
|
||||
window: long
|
||||
annotations:
|
||||
summary: API error budget burning over 6x in both 60s and 120s windows
|
||||
runbook: See infra/runbooks/incident.md#http-errors
|
||||
- alert: Lab6ReadyBurnShort
|
||||
expr: |
|
||||
((sum(rate(messenger_attachments_ready_duration_seconds_count[15s]))
|
||||
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[15s])))
|
||||
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[15s])),0.001) > 0.72)
|
||||
and
|
||||
((sum(rate(messenger_attachments_ready_duration_seconds_count[30s]))
|
||||
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s])))
|
||||
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001) > 0.72)
|
||||
for: 10s
|
||||
labels:
|
||||
severity: page
|
||||
slo: ready
|
||||
window: short
|
||||
annotations:
|
||||
summary: Ready-time error budget burning over 14.4x in both 15s and 30s windows
|
||||
runbook: See infra/runbooks/incident.md#queue-lag
|
||||
- alert: Lab6ReadyBurnLong
|
||||
expr: |
|
||||
((sum(rate(messenger_attachments_ready_duration_seconds_count[60s]))
|
||||
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[60s])))
|
||||
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[60s])),0.001) > 0.30)
|
||||
and
|
||||
((sum(rate(messenger_attachments_ready_duration_seconds_count[120s]))
|
||||
- sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[120s])))
|
||||
/ clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[120s])),0.001) > 0.30)
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
slo: ready
|
||||
window: long
|
||||
annotations:
|
||||
summary: Ready-time error budget burning over 6x in both 60s and 120s windows
|
||||
runbook: See infra/runbooks/incident.md#queue-lag
|
||||
@@ -0,0 +1,54 @@
|
||||
# Lab 6: local incident runbook
|
||||
|
||||
Use Grafana at `http://localhost:3000` (dashboard **Lab 6 · Messenger signals**) or
|
||||
Prometheus rules at `/alerts` inside the Compose network. All times below are
|
||||
UTC. The dashboard uses stored Prometheus data; Tempo and Loki are separate
|
||||
datasources. Keep the last hour visible before diagnosing a recovered incident.
|
||||
|
||||
## HTTP errors
|
||||
|
||||
`Lab6HighHttpErrorRate` fires when 5xx / all non-health HTTP requests exceeds
|
||||
10% over 30 seconds for another 10 seconds. Client 4xx are in the denominator
|
||||
and have a separate panel; they are not server failures. Check HTTP RPS first:
|
||||
an empty or tiny denominator makes an error ratio unreliable. Compare 5xx
|
||||
ratio with DB pool used/waiting and `up` for the telemetry stores. In Tempo,
|
||||
find an `HTTP` server span in the time range and inspect its `db.*` child span
|
||||
duration/status. Query Loki `{service_name="messenger-api"}` for the same
|
||||
trace ID. A growing pool wait and failing DB span identify a database incident;
|
||||
a long server span with fast DB children points to the application path.
|
||||
Restart PostgreSQL only after confirming its container state and persistent
|
||||
volume. Confirm `up`, 5xx ratio, and `/health/ready` recover; record first
|
||||
`pending`, `firing`, and `inactive` timestamps from the rule API.
|
||||
|
||||
## Queue lag
|
||||
|
||||
`Lab6QueueLag` fires when the oldest durable queued job exceeds 10 seconds for
|
||||
another 10 seconds. `messenger_queue_depth` counts rows with `status='queued'`
|
||||
in PostgreSQL, including an outbox row awaiting publication. It is sampled by
|
||||
the publisher. First compare queue depth, oldest age, `messenger_worker_busy`
|
||||
and `messenger_worker_capacity`. If busy disappears and capacity is absent,
|
||||
inspect the worker container; if capacity is present but processing outcome
|
||||
rates stop, inspect broker connectivity and `job_published` events in Loki.
|
||||
If jobs fail repeatedly, inspect `messenger_jobs_processed_total` grouped by
|
||||
outcome and the corresponding consumer trace; do not replay until the cause
|
||||
is corrected. Restart the worker, then verify queued count/age fall, ready
|
||||
durations appear, and the alert returns to inactive.
|
||||
|
||||
## Follow an upload
|
||||
|
||||
Take the `trace_id` from the API `attachment_uploaded` log in Loki. Tempo
|
||||
should contain the API upload span, publisher `attachments.publish` producer
|
||||
span, worker `attachments.process` consumer span and its S3/render/DB children.
|
||||
The worker `job_processed` log has the same trace ID. Job identifiers are
|
||||
synthetic test identifiers; metric labels contain only route templates,
|
||||
status classes, job type/outcome and queue name. Never copy request bodies,
|
||||
tokens, signed object URLs or raw image bytes into evidence.
|
||||
|
||||
## Missing telemetry
|
||||
|
||||
If a signal is absent, check Collector `otelcol_exporter_queue_size`,
|
||||
`otelcol_exporter_send_failed_*` and store `up` before concluding the business
|
||||
service recovered. The SDK uses asynchronous bounded queues and 2-second
|
||||
export timeouts; Collector queues are finite. Export loss during a backend
|
||||
outage is expected and must be reported. A missing sample is not zero errors.
|
||||
Telemetry stores are not part of API readiness.
|
||||
@@ -30,7 +30,7 @@ def main() -> None:
|
||||
write("postgres_password", password)
|
||||
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
|
||||
if not (ROOT / "minio_root_user").exists():
|
||||
write("minio_root_user", "lab5admin")
|
||||
write("minio_root_user", "lab6admin")
|
||||
write("minio_root_password", secrets.token_urlsafe(32))
|
||||
write("s3_access_key_id", secrets.token_hex(10))
|
||||
write("s3_secret_access_key", secrets.token_urlsafe(32))
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture local Prometheus alert states and SLI values without credentials."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
QUERIES = {
|
||||
"http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)',
|
||||
"queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})',
|
||||
"ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)',
|
||||
"collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)',
|
||||
}
|
||||
|
||||
|
||||
def fetch(base, path, params=None):
|
||||
url = base + path + ("?" + urlencode(params) if params else "")
|
||||
with urlopen(url, timeout=4) as response:
|
||||
result = json.load(response)
|
||||
if result["status"] != "success":
|
||||
raise RuntimeError(result)
|
||||
return result["data"]
|
||||
|
||||
|
||||
def snapshot(base):
|
||||
groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"]
|
||||
alerts = [rule for group in groups for rule in group["rules"]]
|
||||
row = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"alerts": {item["name"]: item["state"]
|
||||
for item in alerts if item["name"].startswith("Lab6")},
|
||||
"values": {},
|
||||
}
|
||||
for name, query in QUERIES.items():
|
||||
values = fetch(base, "/api/v1/query", {"query": query})["result"]
|
||||
row["values"][name] = float(values[0]["value"][1]) if values else None
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--prometheus", default="http://127.0.0.1:9090")
|
||||
parser.add_argument("--duration", type=int, default=60)
|
||||
parser.add_argument("--interval", type=int, default=5)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30:
|
||||
parser.error("invalid duration or interval")
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
end = time.monotonic() + args.duration
|
||||
with path.open("w", encoding="utf-8") as out:
|
||||
while True:
|
||||
row = snapshot(args.prometheus)
|
||||
out.write(json.dumps(row, sort_keys=True) + "\n")
|
||||
out.flush()
|
||||
if time.monotonic() + args.interval > end:
|
||||
break
|
||||
time.sleep(args.interval)
|
||||
print(f"saved {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -26,6 +26,8 @@ def pool() -> ConnectionPool:
|
||||
open=True,
|
||||
)
|
||||
atexit.register(result.close)
|
||||
from .telemetry import register_pool_metrics
|
||||
register_pool_metrics(result)
|
||||
return result
|
||||
|
||||
|
||||
|
||||
+29
-1
@@ -8,7 +8,8 @@ from pathlib import Path
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from opentelemetry import trace
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from psycopg_pool import PoolTimeout
|
||||
@@ -24,6 +25,32 @@ tracer = trace.get_tracer(__name__)
|
||||
trace_context = TraceContextTextMapPropagator()
|
||||
|
||||
|
||||
def register_queue_metrics():
|
||||
"""Sample the durable backlog; broker delivery does not erase queued jobs."""
|
||||
meter = metrics.get_meter("messenger.queue")
|
||||
|
||||
def backlog(_):
|
||||
try:
|
||||
with pool().connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS depth,"
|
||||
"coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS oldest_age "
|
||||
"FROM attachment_jobs WHERE status='queued'"
|
||||
).fetchone()
|
||||
return row
|
||||
except (psycopg.Error, PoolTimeout, OSError):
|
||||
return None
|
||||
|
||||
def observe(field):
|
||||
def callback(options):
|
||||
row = backlog(options)
|
||||
return [Observation(row[field], {"queue_name": QUEUE})] if row is not None else []
|
||||
return callback
|
||||
|
||||
meter.create_observable_gauge("messenger.queue.depth", callbacks=[observe("depth")])
|
||||
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[observe("oldest_age")], unit="s")
|
||||
|
||||
|
||||
def publish_batch(channel):
|
||||
with pool().connection() as conn:
|
||||
token, rows = claim_outbox(conn, limit=10, lease_seconds=30)
|
||||
@@ -54,6 +81,7 @@ def publish_batch(channel):
|
||||
|
||||
def main():
|
||||
configure("messenger-publisher")
|
||||
register_queue_metrics()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.getLogger("pika").setLevel(logging.ERROR)
|
||||
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
@@ -22,6 +24,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10)
|
||||
JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120)
|
||||
READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -53,6 +56,7 @@ def configure(service_name: str) -> bool:
|
||||
views=[
|
||||
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
||||
View(instrument_name="messenger.jobs.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.attachments.ready_duration", aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
|
||||
],
|
||||
))
|
||||
|
||||
@@ -65,6 +69,15 @@ def configure(service_name: str) -> bool:
|
||||
app_logger = logging.getLogger("src")
|
||||
app_logger.setLevel(logging.INFO)
|
||||
app_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs))
|
||||
meter = metrics.get_meter("messenger.runtime")
|
||||
meter.create_observable_counter("messenger.process.cpu.time", callbacks=[
|
||||
lambda _: [Observation(time.process_time())],
|
||||
], unit="s")
|
||||
def rss(_):
|
||||
with open("/proc/self/statm", encoding="ascii") as statm:
|
||||
return [Observation(int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))]
|
||||
|
||||
meter.create_observable_gauge("messenger.process.rss", callbacks=[rss], unit="By")
|
||||
return True
|
||||
|
||||
|
||||
@@ -84,3 +97,32 @@ def job_instruments():
|
||||
meter.create_counter("messenger.jobs.processed", unit="1"),
|
||||
meter.create_histogram("messenger.jobs.duration", unit="s"),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def pipeline_instruments():
|
||||
meter = metrics.get_meter("messenger.pipeline")
|
||||
return (
|
||||
meter.create_histogram("messenger.attachments.ready_duration", unit="s"),
|
||||
meter.create_up_down_counter("messenger.worker.busy"),
|
||||
)
|
||||
|
||||
|
||||
def register_worker_capacity():
|
||||
metrics.get_meter("messenger.pipeline").create_observable_gauge(
|
||||
"messenger.worker.capacity", callbacks=[lambda _: [Observation(1)]],
|
||||
)
|
||||
|
||||
|
||||
def register_pool_metrics(pool):
|
||||
"""Read local pool counters without querying the database."""
|
||||
meter = metrics.get_meter("messenger.pool")
|
||||
|
||||
def observe(_):
|
||||
stats = pool.get_stats()
|
||||
return [Observation(stats["pool_size"] - stats["pool_available"])]
|
||||
|
||||
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[observe])
|
||||
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[
|
||||
lambda _: [Observation(pool.get_stats()["requests_waiting"])],
|
||||
])
|
||||
|
||||
+10
-2
@@ -9,7 +9,7 @@ import re
|
||||
import signal
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
@@ -26,7 +26,7 @@ from .attachments import BUCKET, storage
|
||||
from .db import pool
|
||||
from .image_variants import InvalidImage, MAX_BYTES, render
|
||||
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
|
||||
from .telemetry import configure, job_instruments
|
||||
from .telemetry import configure, job_instruments, pipeline_instruments, register_worker_capacity
|
||||
|
||||
|
||||
logger = logging.getLogger("src.worker")
|
||||
@@ -151,6 +151,10 @@ def process_message(item: dict) -> str:
|
||||
with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False):
|
||||
current = complete_job(conn, job["id"], token, stored)
|
||||
_fault("after_result_commit_before_ack", job["id"])
|
||||
if current:
|
||||
ready_duration, _ = pipeline_instruments()
|
||||
ready_duration.record(max(0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
|
||||
{"job_type": "attachment.process.v1"})
|
||||
if current and job["staging_key"]:
|
||||
try:
|
||||
storage().remove_object(BUCKET, job["staging_key"])
|
||||
@@ -194,6 +198,8 @@ def traced_process_message(item: dict) -> str:
|
||||
span.set_attribute("job.id", item["job_id"])
|
||||
span.set_attribute("attachment.id", item["attachment_id"])
|
||||
started = time.monotonic()
|
||||
_, busy = pipeline_instruments()
|
||||
busy.add(1)
|
||||
outcome = "error"
|
||||
try:
|
||||
outcome = process_message(item)
|
||||
@@ -202,6 +208,7 @@ def traced_process_message(item: dict) -> str:
|
||||
outcome = "database_unavailable"
|
||||
raise
|
||||
finally:
|
||||
busy.add(-1)
|
||||
attributes = {"job_type": item["type"], "outcome": outcome}
|
||||
processed, duration = job_instruments()
|
||||
processed.add(1, attributes)
|
||||
@@ -215,6 +222,7 @@ def traced_process_message(item: dict) -> str:
|
||||
|
||||
def main() -> None:
|
||||
configure("messenger-worker")
|
||||
register_worker_capacity()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
|
||||
raise SystemExit("invalid worker retry settings")
|
||||
|
||||
Reference in New Issue
Block a user