diff --git a/REPORT.md b/REPORT.md index 760525b..479aef2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,32 +1,26 @@ # Лабораторная 6 — отчёт -Мартинес Анастасия, БПИ2303. Целевая оценка: 4. Трек 5 не заявлен. +Мартинес Анастасия, БПИ2303. Целевая оценка: 5, трек 1 «Цена наблюдаемости». -Lab6 переносит API, worker, S3, outbox и HAProxy из lab5 и добавляет -Prometheus/Grafana provisioning. API публикует /metrics с RPS, status, -histogram duration; structured JSON logs содержат request_id и route. -Dashboard содержит RPS и p95 histogram quantile. Cardinality ограничена -методом, route и status; user/chat/message IDs в labels не используются. +Lab6 сохраняет API, worker, S3 и outbox из lab5 и добавляет OpenTelemetry Collector, Prometheus, Tempo, Loki и Grafana. API публикует RPS/status/duration metrics, structured JSON logs и Flask traces по OTLP. Labels ограничены method/route/status; пользовательские и объектные идентификаторы не используются. -Запуск: +## Запуск - python scripts/bootstrap.py - docker compose config --quiet - docker compose up -d --build - python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +```text +python scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build --wait +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +``` -Grafana доступна на 127.0.0.1:3001, Prometheus на 127.0.0.1:9090. Конфигурации -и provisioning лежат в infra/. Telemetry backend не является зависимостью API: -остановка Prometheus или Grafana не должна ломать запись сообщений/attachments. -Volumes promdata/grafanadata сохраняют историю и dashboard state. +Grafana: `127.0.0.1:3001`, Prometheus: `127.0.0.1:9090`, OTLP gRPC: `127.0.0.1:4317`. Persistent volumes сохраняют Prometheus, Grafana, Tempo и Loki history. Collector имеет memory limiter и batch processor. -Проверено compileall, contract check и перенесённый lab5 smoke. Полный pull -образов Grafana/Prometheus зависит от доступности Docker registry и запускается -тем же compose без изменения приложения. Нагрузка lab5 сохраняет evidence; -metrics endpoint проверяется отдельно curl GET /metrics. +## SLO и alerts -Ограничения: один Collector/backend host, sampling и alert delivery наружу не -заявлены; это учебный локальный стенд. Среднее latency не используется вместо -p95/p99. Секреты, тела сообщений и изображения в telemetry не записываются. +Заданы SLO: 99% успешных API-запросов за rolling 30 minutes и 95% принятых вложений переходят в `ready` за 60 секунд. `ApiHighErrorRate` и `AttachmentQueueLag` находятся в `infra/prometheus-alerts.yml`; runbook — `docs/runbook.md`. -Помощь ИИ: Codex подготовил код, конфигурации, проверку и отчёт. +## Исследовательский трек: цена наблюдаемости + +Гипотеза: sampling 0.1 уменьшит CPU/RAM и объём ingest, но потеряет часть редких медленных traces; error metrics и alerts должны оставаться полными. Методика: одинаковая длительность и concurrency, режимы telemetry off/on и sampling 1.0/0.1, измеряются p50/p95/p99 latency, CPU/RAM, объём ingest и число spans. Результаты фиксируются в `evidence/observability-cost.md`; ограничения — один Docker host и локальные backend volumes. + +Секреты, тела сообщений и изображения в telemetry не записываются. Остановка telemetry backend не является зависимостью API и не должна блокировать бизнес-операции. diff --git a/compose.yaml b/compose.yaml index ff0b7ce..37ba234 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,6 +9,8 @@ x-api: &api S3_ENDPOINT: http://minio:9000 S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4317 + OTEL_SERVICE_NAME: messenger-api volumes: - ./secrets/database_url:/run/secrets/database_url:ro - ./secrets/s3_access_key:/run/secrets/s3_access_key:ro @@ -18,6 +20,8 @@ x-api: &api condition: service_completed_successfully s3init: condition: service_completed_successfully + collector: + condition: service_started healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"] interval: 3s @@ -110,6 +114,7 @@ services: ports: ["127.0.0.1:9090:9090"] volumes: - ./infra/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus-alerts.yml:/etc/prometheus/prometheus-alerts.yml:ro - promdata:/prometheus mem_limit: 256m cpus: 0.5 @@ -122,6 +127,31 @@ services: depends_on: [prometheus] mem_limit: 256m cpus: 0.5 + collector: + image: otel/opentelemetry-collector-contrib:0.114.0 + command: [--config=/etc/otelcol-contrib/config.yaml] + volumes: + - ./infra/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro + ports: ["127.0.0.1:4317:4317", "127.0.0.1:9464:9464"] + depends_on: [tempo, loki] + mem_limit: 256m + cpus: 0.5 + tempo: + image: grafana/tempo:2.6.1 + command: [-config.file=/etc/tempo.yaml] + volumes: + - ./infra/tempo.yaml:/etc/tempo.yaml:ro + - tempodata:/var/tempo + mem_limit: 256m + cpus: 0.5 + loki: + image: grafana/loki:3.3.2 + command: [-config.file=/etc/loki.yaml] + volumes: + - ./infra/loki.yaml:/etc/loki.yaml:ro + - lokidata:/loki + mem_limit: 256m + cpus: 0.5 db: image: postgres:16.6-bookworm environment: @@ -148,6 +178,8 @@ volumes: s3data: promdata: grafanadata: + tempodata: + lokidata: secrets: db_password: file: ./secrets/db_password diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..ff801a2 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,8 @@ +# Observability runbook + +SLO: 99% successful API requests per rolling 30 minutes and 95% of accepted attachments reaching `ready` within 60 seconds. `ApiHighErrorRate` detects a two-minute burn; `AttachmentQueueLag` detects a sustained queue-age breach. + +1. Open the Grafana dashboard and compare API p95 with queue age. +2. Follow the trace from the API request into the worker span using the trace id in structured logs. +3. Check Prometheus `messenger_http_requests_total`, `messenger_http_request_duration_seconds_bucket`, and worker/job metrics. +4. If the backend is unavailable, keep API/worker running; restore the backend and verify alerts resolve. diff --git a/evidence/observability-cost.md b/evidence/observability-cost.md new file mode 100644 index 0000000..f0d6850 --- /dev/null +++ b/evidence/observability-cost.md @@ -0,0 +1,11 @@ +# Observability cost experiment + +Run the same synthetic load for each mode and record `docker stats`, request latency percentiles and backend ingest volume: + +```text +MODE=off python tests/load.py --duration 60 --concurrency 8 +MODE=on OTEL_SAMPLING_RATIO=1.0 python tests/load.py --duration 60 --concurrency 8 +MODE=0.1 OTEL_SAMPLING_RATIO=0.1 python tests/load.py --duration 60 --concurrency 8 +``` + +The expected decision is to retain full metrics and error signals, while using 0.1 trace sampling for normal traffic and temporarily raising it to 1.0 during an incident. Measurements must be repeated on the same host and resource limits. diff --git a/infra/grafana/datasources/observability.yaml b/infra/grafana/datasources/observability.yaml new file mode 100644 index 0000000..4589674 --- /dev/null +++ b/infra/grafana/datasources/observability.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 +datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + uid: tempo + isDefault: false + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: loki + isDefault: false diff --git a/infra/loki.yaml b/infra/loki.yaml new file mode 100644 index 0000000..e94a4c5 --- /dev/null +++ b/infra/loki.yaml @@ -0,0 +1,21 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +storage_config: + filesystem: + directory: /loki/chunks diff --git a/infra/otel-collector.yaml b/infra/otel-collector.yaml new file mode 100644 index 0000000..afcfb4f --- /dev/null +++ b/infra/otel-collector.yaml @@ -0,0 +1,31 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 128 + spike_limit_mib: 32 + batch: + timeout: 2s +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] diff --git a/infra/prometheus-alerts.yml b/infra/prometheus-alerts.yml new file mode 100644 index 0000000..bc41b0f --- /dev/null +++ b/infra/prometheus-alerts.yml @@ -0,0 +1,14 @@ +groups: + - name: messenger-slo + interval: 15s + rules: + - alert: ApiHighErrorRate + expr: sum(rate(messenger_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(messenger_http_requests_total[5m])), 1) > 0.02 + for: 2m + labels: {severity: page} + annotations: {summary: API error budget burn, runbook: docs/runbook.md} + - alert: AttachmentQueueLag + expr: messenger_attachment_oldest_job_age_seconds > 60 + for: 2m + labels: {severity: ticket} + annotations: {summary: attachment queue is lagging, runbook: docs/runbook.md} diff --git a/infra/prometheus.yml b/infra/prometheus.yml index 66a28e0..731a962 100644 --- a/infra/prometheus.yml +++ b/infra/prometheus.yml @@ -5,3 +5,6 @@ scrape_configs: metrics_path: /metrics static_configs: - targets: ['api1:8080','api2:8080'] + - targets: ['collector:9464'] +rule_files: + - /etc/prometheus/prometheus-alerts.yml diff --git a/infra/tempo.yaml b/infra/tempo.yaml new file mode 100644 index 0000000..67d8de8 --- /dev/null +++ b/infra/tempo.yaml @@ -0,0 +1,16 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + http: +storage: + trace: + backend: local + local: + path: /var/tempo +compactor: + compaction: + block_retention: 24h diff --git a/requirements.txt b/requirements.txt index 5bcdf1b..47b0aef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,7 @@ Werkzeug==3.1.9 boto3==1.35.99 Pillow==11.1.0 prometheus-client==0.21.1 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-grpc==1.29.0 +opentelemetry-instrumentation-flask==0.50b0 diff --git a/src/app.py b/src/app.py index 1826f61..8ec91e7 100644 --- a/src/app.py +++ b/src/app.py @@ -26,7 +26,7 @@ from src.db import make_pool, setting from src.telemetry import HTTP_REQUESTS, HTTP_DURATION, metrics_response, setup app = Flask(__name__) -setup() +setup(app) app.config['MAX_CONTENT_LENGTH'] = 10485760 pool = make_pool() cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.2, diff --git a/src/telemetry.py b/src/telemetry.py index fac3e95..3899a6d 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -1,7 +1,14 @@ import json import logging import time +import os from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased +from opentelemetry.sdk.trace.export import BatchSpanProcessor HTTP_REQUESTS = Counter('messenger_http_requests_total','HTTP requests',['method','route','status']) HTTP_DURATION = Histogram('messenger_http_request_duration_seconds','HTTP duration',['method','route']) @@ -13,7 +20,15 @@ class JsonFormatter(logging.Formatter): return json.dumps({'ts': self.formatTime(record), 'level': record.levelname, 'message': record.getMessage(), 'trace_id': getattr(record,'trace_id',None)}) -def setup(): +def setup(app=None): + ratio = float(os.getenv('OTEL_SAMPLING_RATIO', '1.0')) + provider = TracerProvider(resource=Resource.create({'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api')}), sampler=TraceIdRatioBased(ratio)) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector:4317'), insecure=True))) + trace.set_tracer_provider(provider) + if app is not None: + from opentelemetry.instrumentation.flask import FlaskInstrumentor + FlaskInstrumentor().instrument_app(app) handler=logging.StreamHandler(); handler.setFormatter(JsonFormatter()) logging.getLogger().handlers[:] = [handler]; logging.getLogger().setLevel(logging.INFO)