From 4bf383bd519cbd2caaafe45f030854ea2d50cfb4 Mon Sep 17 00:00:00 2001 From: dizi Date: Sun, 1 Nov 2026 17:06:35 +0300 Subject: [PATCH] ops: add observability stack and SLO alerts --- Dockerfile | 3 +- compose.yaml | 103 ++- infra/loki/loki.yaml | 48 ++ infra/otel/collector.yaml | 73 +++ infra/tempo/tempo.yaml | 48 ++ lab5.env.example => lab6.env.example | 21 +- .../{messenger.json => messenger-lab6.json} | 610 +++++++++++++++++- .../grafana/provisioning/alerting/.gitkeep | 1 + .../datasources/observability.yml | 74 +++ .../provisioning/datasources/prometheus.yml | 10 - .../grafana/provisioning/plugins/.gitkeep | 1 + monitoring/prometheus.yml | 46 +- monitoring/rules/messenger.yml | 178 +++++ monitoring/runbooks/api-availability.md | 72 +++ monitoring/runbooks/attachment-pipeline.md | 85 +++ 15 files changed, 1341 insertions(+), 32 deletions(-) create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/tempo/tempo.yaml rename lab5.env.example => lab6.env.example (80%) rename monitoring/grafana/dashboards/{messenger.json => messenger-lab6.json} (55%) create mode 100644 monitoring/grafana/provisioning/alerting/.gitkeep create mode 100644 monitoring/grafana/provisioning/datasources/observability.yml delete mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/grafana/provisioning/plugins/.gitkeep create mode 100644 monitoring/rules/messenger.yml create mode 100644 monitoring/runbooks/api-availability.md create mode 100644 monitoring/runbooks/attachment-pipeline.md diff --git a/Dockerfile b/Dockerfile index efd6b0d..ab37655 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,9 @@ ENV SWAGGER_JSON=/spec/openapi.json FROM prom/prometheus:v3.13.3 AS prometheus COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml +COPY monitoring/rules /etc/prometheus/rules -FROM grafana/grafana:13.1.6 AS grafana +FROM grafana/grafana:13.2.4 AS grafana COPY monitoring/grafana/provisioning /etc/grafana/provisioning COPY monitoring/grafana/dashboards /etc/grafana/dashboards diff --git a/compose.yaml b/compose.yaml index 291eaec..d21c2eb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume target: /etc/messenger/messenger.yaml read_only: true +x-telemetry-environment: &telemetry-environment + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL} + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION} + services: postgres: image: postgres:17.6-alpine3.22 @@ -118,7 +123,7 @@ services: networks: [data] migrate: - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/messenger", "migrate"] environment: MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml @@ -137,8 +142,10 @@ services: api: build: . - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-api MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml APP_HOST: ${APP_HOST} APP_PORT: ${APP_PORT} @@ -176,9 +183,11 @@ services: networks: [application, data] dispatcher: - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/messenger", "dispatch"] environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-dispatcher MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} BROKER_URL: ${BROKER_URL} @@ -202,9 +211,11 @@ services: networks: [data, monitoring] worker: - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/messenger", "worker"] environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-worker MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml DATABASE_URL: ${DATABASE_URL} S3_ENDPOINT: ${S3_ENDPOINT} @@ -239,7 +250,7 @@ services: build: context: . target: loadbalancer - image: ${NGINX_IMAGE:-mtusi-nginx:lab5} + image: ${NGINX_IMAGE:-mtusi-nginx:lab6} ports: - "${API_BIND_ADDRESS}:8080" depends_on: @@ -259,7 +270,7 @@ services: build: context: . target: swagger - image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5} + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6} ports: - "${SWAGGER_BIND_ADDRESS}:8080" depends_on: @@ -289,7 +300,7 @@ services: lab5-experiment: profiles: ["experiments"] - image: ${API_IMAGE:-mtusi-messenger:lab5} + image: ${API_IMAGE:-mtusi-messenger:lab6} command: ["/app/lab5experiment"] environment: MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml @@ -375,11 +386,12 @@ services: build: context: . target: prometheus - image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5} + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6} command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=7d + - --web.enable-remote-write-receiver volumes: - prometheus_data:/prometheus ports: @@ -398,6 +410,9 @@ services: - dispatcher - worker - s3 + - otel-collector + - loki + - tempo cpus: ${PROMETHEUS_CPUS} mem_limit: ${PROMETHEUS_MEMORY} restart: unless-stopped @@ -407,28 +422,98 @@ services: build: context: . target: grafana - image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5} + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6} environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} GF_USERS_ALLOW_SIGN_UP: "false" + GF_EXPLORE_ENABLED: "true" volumes: - grafana_data:/var/lib/grafana ports: - "${GRAFANA_BIND_ADDRESS}:3000" depends_on: - prometheus + - loki + - tempo + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"] + interval: 5s + timeout: 3s + retries: 20 cpus: ${GRAFANA_CPUS} mem_limit: ${GRAFANA_MEMORY} restart: unless-stopped networks: [edge, monitoring] + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + ports: + - "${LOKI_BIND_ADDRESS}:3100" + healthcheck: + test: ["CMD", "/usr/bin/loki", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${LOKI_CPUS} + mem_limit: ${LOKI_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + tempo: + image: grafana/tempo:2.10.10 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + ports: + - "${TEMPO_BIND_ADDRESS}:3200" + healthcheck: + test: ["CMD", "/tempo", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${TEMPO_CPUS} + mem_limit: ${TEMPO_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.160.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + expose: + - "4317" + - "4318" + - "8888" + - "8889" + - "13133" + depends_on: + - loki + - tempo + healthcheck: + test: ["CMD", "/otelcol-contrib", "components"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${OTEL_COLLECTOR_CPUS} + mem_limit: ${OTEL_COLLECTOR_MEMORY} + restart: unless-stopped + networks: [application, data, monitoring] + volumes: postgres_data: valkey_data: redis_broker_data: prometheus_data: grafana_data: + loki_data: + tempo_data: s3_data: networks: diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..4c04d5d --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,48 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + volume_enabled: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - service.name + - deployment.environment.name + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e912195 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,73 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_constant_labels: + included: + - service.name + - service.version + - deployment.environment.name + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_grpc/tempo] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..0d8a229 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,48 @@ +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9095 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +metrics_generator: + registry: + external_labels: + source: tempo + deployment_environment: lab + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + +usage_report: + reporting_enabled: false diff --git a/lab5.env.example b/lab6.env.example similarity index 80% rename from lab5.env.example rename to lab6.env.example index 5d3809a..ca34a40 100644 --- a/lab5.env.example +++ b/lab6.env.example @@ -1,3 +1,6 @@ +# Изолированное имя Compose-проекта и его volumes/networks. +COMPOSE_PROJECT_NAME=mtusi2026lab6 + # Адрес и внутренний порт HTTP-сервера. APP_HOST=0.0.0.0 APP_PORT=8080 @@ -8,6 +11,14 @@ API_BIND_ADDRESS=127.0.0.1:8080 SWAGGER_BIND_ADDRESS=127.0.0.1:8081 PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 GRAFANA_BIND_ADDRESS=127.0.0.1:3000 +LOKI_BIND_ADDRESS=127.0.0.1:3100 +TEMPO_BIND_ADDRESS=127.0.0.1:3200 + +# Ресурсные атрибуты и внутренний OTLP endpoint приложений. +DEPLOYMENT_ENVIRONMENT=lab +SERVICE_VERSION=lab6 +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc # Число API-реплик за Nginx. API_REPLICAS=2 @@ -55,8 +66,14 @@ EXPORTER_CPUS=0.25 EXPORTER_MEMORY=128m PROMETHEUS_CPUS=1 PROMETHEUS_MEMORY=512m -GRAFANA_CPUS=0.5 -GRAFANA_MEMORY=256m +GRAFANA_CPUS=1 +GRAFANA_MEMORY=512m +OTEL_COLLECTOR_CPUS=0.5 +OTEL_COLLECTOR_MEMORY=256m +LOKI_CPUS=1 +LOKI_MEMORY=512m +TEMPO_CPUS=1 +TEMPO_MEMORY=512m POSTGRES_CPUS=2 POSTGRES_MEMORY=1g VALKEY_CPUS=1 diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger-lab6.json similarity index 55% rename from monitoring/grafana/dashboards/messenger.json rename to monitoring/grafana/dashboards/messenger-lab6.json index de0ef2d..f20cabe 100644 --- a/monitoring/grafana/dashboards/messenger.json +++ b/monitoring/grafana/dashboards/messenger-lab6.json @@ -67,7 +67,7 @@ }, { "type": "timeseries", - "title": "API latency p95 by route", + "title": "API latency p50 / p95 / p99 by route", "id": 3, "gridPos": { "h": 8, @@ -77,9 +77,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", - "legendFormat": "{{method}} {{route}}", + "expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50 {{method}} {{route}}", "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95 {{method}} {{route}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99 {{method}} {{route}}", + "refId": "C" } ], "fieldConfig": { @@ -895,14 +905,563 @@ } } ] + }, + { + "type": "row", + "title": "SLO and alerts", + "id": 105, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 109 + }, + "panels": [ + { + "type": "timeseries", + "title": "Business API availability", + "id": 31, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 110 + }, + "targets": [ + { + "expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "availability", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "API error-budget burn rate (99.5% SLO)", + "id": 32, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 110 + }, + "targets": [ + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachments ready within 30 seconds", + "id": 33, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 118 + }, + "targets": [ + { + "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "ready in 30s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachment ready-time error-budget burn", + "id": 34, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 118 + }, + "targets": [ + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "SLO alert state", + "id": 35, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 126 + }, + "targets": [ + { + "expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})", + "legendFormat": "{{alertname}} {{alertstate}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Runtime resources", + "id": 107, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 134 + }, + "panels": [ + { + "type": "timeseries", + "title": "Process CPU by service", + "id": 38, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Go heap by service", + "id": 39, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Database pool used and wait rate", + "id": 40, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 143 + }, + "targets": [ + { + "expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})", + "legendFormat": "{{service_name}} used", + "refId": "A" + }, + { + "expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))", + "legendFormat": "{{service_name}} waits/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Worker saturation", + "id": 41, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 143 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "busy", + "refId": "A" + }, + { + "expr": "sum(messenger_attachment_worker_capacity)", + "legendFormat": "capacity", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Logs and traces", + "id": 106, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 151 + }, + "panels": [ + { + "type": "logs", + "title": "Correlated application logs", + "id": 36, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 152 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=~\"messenger-.+\"} | json", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + } + }, + { + "type": "traces", + "title": "Recent application traces (TraceQL)", + "id": 37, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 152 + }, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "query": "{ resource.service.name =~ \"messenger-.*\" }", + "queryType": "traceql", + "refId": "A", + "limit": 50, + "tableType": "traces" + } + ], + "options": { + "showSpanFilters": true + } + }, + { + "type": "timeseries", + "title": "Trace-derived service call rate", + "id": 42, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))", + "legendFormat": "{{service}} {{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "type": "timeseries", + "title": "Collector delivery failures and queue pressure", + "id": 43, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))", + "legendFormat": "{{exporter}} spans failed/s", + "refId": "A" + }, + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))", + "legendFormat": "{{exporter}} logs failed/s", + "refId": "B" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "{{exporter}} queue", + "refId": "C" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_capacity)", + "legendFormat": "{{exporter}} capacity", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] } ], "refresh": "5s", "schemaVersion": 41, "tags": [ "highload", - "lab5", - "pipeline" + "lab6", + "observability", + "slo" ], "templating": { "list": [] @@ -912,7 +1471,42 @@ "to": "now" }, "timezone": "browser", - "title": "Messenger lab5 overview", - "uid": "messenger-lab5", - "version": 3 + "title": "Messenger lab6 observability", + "uid": "messenger-lab6", + "version": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Explore logs and traces", + "type": "link", + "url": "/explore" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "API runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Pipeline runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md" + } + ] } diff --git a/monitoring/grafana/provisioning/alerting/.gitkeep b/monitoring/grafana/provisioning/alerting/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/grafana/provisioning/datasources/observability.yml b/monitoring/grafana/provisioning/datasources/observability.yml new file mode 100644 index 0000000..99af68b --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/observability.yml @@ -0,0 +1,74 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - name: TraceID + matcherRegex: 'trace_id[=:" ]+([0-9a-f]{32})' + datasourceUid: tempo + url: '$${__value.raw}' + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + nodeGraph: + enabled: true + search: + hide: false + serviceMap: + datasourceUid: prometheus + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"' + tags: + - key: service.name + value: service_name + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: service + queries: + - name: Request rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))' + - name: Error rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))' diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml deleted file mode 100644 index 8704d9e..0000000 --- a/monitoring/grafana/provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - uid: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: false diff --git a/monitoring/grafana/provisioning/plugins/.gitkeep b/monitoring/grafana/provisioning/plugins/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/plugins/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 1869184..70b5dfd 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -1,6 +1,11 @@ global: - scrape_interval: 1s - evaluation_interval: 1s + scrape_interval: 2s + evaluation_interval: 5s + external_labels: + deployment_environment: lab + +rule_files: + - /etc/prometheus/rules/*.yml scrape_configs: - job_name: messenger @@ -9,6 +14,13 @@ scrape_configs: type: A port: 8080 refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-api + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep - job_name: nginx static_configs: @@ -32,12 +44,42 @@ scrape_configs: type: A port: 9091 refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-worker + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep - job_name: outbox-dispatcher static_configs: - targets: ["dispatcher:9091"] + relabel_configs: + - target_label: service_name + replacement: messenger-dispatcher + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep - job_name: minio metrics_path: /minio/v2/metrics/cluster static_configs: - targets: ["s3:9000"] + + - job_name: otel-applications + static_configs: + - targets: ["otel-collector:8889"] + + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] diff --git a/monitoring/rules/messenger.yml b/monitoring/rules/messenger.yml new file mode 100644 index 0000000..4999ddb --- /dev/null +++ b/monitoring/rules/messenger.yml @@ -0,0 +1,178 @@ +groups: + - name: messenger-api-slo + interval: 5s + rules: + - alert: MessengerAPIBusiness5xxHigh + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0)) + / + clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001) + ) > 0.05 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: Business API 5xx ratio is above 5% + description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnFast + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001) + ) > 0.072 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001) + ) > 0.072 + for: 30s + labels: + severity: critical + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning fast + description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnSlow + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001) + ) > 0.03 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001) + ) > 0.03 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning steadily + description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - name: messenger-attachment-slo + interval: 5s + rules: + - alert: AttachmentReadyHighErrorBudgetBurnFast + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)), + 0.000001 + ) > 0.144 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)), + 0.000001 + ) > 0.144 + for: 30s + labels: + severity: critical + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning fast + description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentReadyHighErrorBudgetBurnSlow + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)), + 0.000001 + ) > 0.06 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)), + 0.000001 + ) > 0.06 + for: 1m + labels: + severity: warning + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning steadily + description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentWorkerUnavailable + expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1 + for: 30s + labels: + severity: critical + service: messenger-worker + annotations: + summary: No attachment worker is available + description: Prometheus has not observed a healthy worker target for 30 seconds. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentQueueBacklogGrowing + expr: | + max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30 + and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0 + for: 30s + labels: + severity: warning + service: messenger-worker + annotations: + summary: Attachment queue backlog is growing + description: The oldest queued attachment is older than 30 seconds and its age continues to increase. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentRedisStreamLagHigh + expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20 + for: 1m + labels: + severity: warning + service: redis-broker + annotations: + summary: Redis Stream consumer lag is high + description: The attachment worker group has more than 20 messages waiting for delivery. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md diff --git a/monitoring/runbooks/api-availability.md b/monitoring/runbooks/api-availability.md new file mode 100644 index 0000000..2ff5453 --- /dev/null +++ b/monitoring/runbooks/api-availability.md @@ -0,0 +1,72 @@ +# API availability SLO + +## Цель и сигналы + +Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%. + +Основные сигналы: + +- `messenger_http_requests_total{method,route,status_class}` — число запросов; +- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency; +- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets; +- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения; +- trace `messenger-api` — путь одного запроса по зависимостям; +- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`. + +`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента. +2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью. +3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler. +4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL: + + ```text + { resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error } + ``` + +5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют. +6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki: + + ```text + {service_name="messenger-api"} | json | trace_id="" + ``` + +7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют. + +## Проверка alert + +Состояние правил видно в Grafana Alerting и в Prometheus: + +```bash +curl -s http://127.0.0.1:9090/api/v1/alerts +``` + +Контролируемый отказ PostgreSQL: + +```bash +ENV_FILE=lab6.env.example scripts/incident-database.sh +``` + +Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются. + +Ожидаемая последовательность: + +```text +нормальный error ratio +→ PostgreSQL остановлен +→ 5xx растёт +→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing +→ PostgreSQL восстановлен +→ успешные ответы возвращаются +→ оперативный alert resolved; burn-rate хранит историю длинного окна +``` + +## Действия + +1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи. +2. Если API targets недоступны, восстановить реплики по одной и проверить readiness. +3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины. +4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline. +5. Не удалять volumes и не сбрасывать очередь ради закрытия alert. diff --git a/monitoring/runbooks/attachment-pipeline.md b/monitoring/runbooks/attachment-pipeline.md new file mode 100644 index 0000000..c2ae007 --- /dev/null +++ b/monitoring/runbooks/attachment-pipeline.md @@ -0,0 +1,85 @@ +# Attachment pipeline SLO + +## Цель и сигналы + +Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget. + +Основные сигналы: + +- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker; +- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки; +- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время; +- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL; +- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job; +- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker; +- `messenger_outbox_pending` — задержка до публикации; +- Redis Stream lag и PEL — доставка и неподтверждённые сообщения. + +Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента. +2. Определить место задержки: + - растёт `messenger_outbox_pending` — проблема до публикации в broker; + - растут stream lag и `queued` — worker не успевает или недоступен; + - растут PEL и `processing` — delivery получена, но не завершена; + - растёт `failed` — постоянные ошибки или исчерпаны retry; + - busy равен capacity — worker насыщен. +3. В Explore → Tempo найти upload по route или worker consumer span: + + ```text + { resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" } + ``` + + ```text + { resource.service.name = "messenger-worker" && name = "attachment.process" } + ``` + +4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label. +5. Перейти из выбранного span в Loki или выполнить: + + ```text + {service_name="messenger-worker"} | json | trace_id="" + ``` + +6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются. + +## Проверка alert + +Короткая деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh short +``` + +Длительная деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh long +``` + +Ожидаемая последовательность: + +```text +worker up, backlog около нуля +→ worker остановлен +→ queued и oldest age растут +→ short: AttachmentWorkerUnavailable firing +→ long: AttachmentQueueBacklogGrowing firing +→ worker запущен +→ backlog дренируется +→ все принятые attachments достигают ready +→ alert resolved +``` + +Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload. + +## Действия + +1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования. +2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency. +3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина. +4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim. +5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины. +6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment.