Files
mtusi2026lab6/compose.yaml
T
celis 51b02a541d Реализовать лабораторную 6: OpenTelemetry, Grafana, SLO и burn-rate alerts
- api и worker отдают traces, metrics и logs по OTLP в Collector
  (memory_limiter, redaction, batch, ограниченная очередь и retry);
  хранилища: Prometheus (OTLP receiver), Loki, Tempo; Grafana с
  provisioning datasources и dashboard «Messenger lab6»
- W3C trace context через outbox и RabbitMQ: producer/consumer spans
  связывают upload, публикации и попытки worker в одном trace
- SLO доступности API (SLI на HAProxy) и времени до ready, multiwindow
  burn-rate alerts, ApiHighErrorRate, JobQueueLagGrowing,
  DbPoolSaturated, TelemetryExportFailing; unit-тесты promtool, runbooks
- кампания трека 2 (k6, 6 отказов по расписанию), разбор и графики;
  проверки сигналов и истории telemetry, evidence и REPORT.md
2026-10-31 01:45:56 +03:00

387 lines
18 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Клиент → lb (HAProxy) → api × N → postgres (метаданные, jobs, outbox) + s3 (SeaweedFS, staging/ оригинала).
# api (relay outbox) → rabbitmq (quorum-очередь attachments.process) → worker × M → s3 (originals/, variants/) + postgres.
# Наблюдаемость (lab6): api/worker → OTLP → otel-collector → prometheus (метрики) / loki (логи) / tempo (трассы) → grafana.
# Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build (см. REPORT.md)
name: messenger-lab6
x-app: &app
build: .
image: messenger-lab6-app:dev # уникальное имя: на общем Docker-host не перепутать с чужими стендами
networks: [backend]
logging: &logging
driver: json-file
options: { max-size: "10m", max-file: "3" } # stdout-логи ограничены; история логов — в Loki
# OpenTelemetry SDK api и worker (src/telemetry.ts). service.name задаётся у каждого сервиса отдельно.
# OTEL_SDK_DISABLED=true — приложение без telemetry (сравнение, отладка).
x-otel-env: &otel-env
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_PROPAGATORS: tracecontext # W3C traceparent/tracestate; baggage клиентов не принимается
OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс; окна rate() от 1 минуты
OTEL_BSP_SCHEDULE_DELAY: "1000"
OTEL_BLRP_SCHEDULE_DELAY: "1000"
OTEL_NODE_EXPERIMENTAL_SDK_METRICS: "true" # otel.sdk.*: сколько spans/логов SDK обработал и отбросил
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
x-jobs-env: &jobs-env
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-2000}
JOB_REPUBLISH_SECONDS: ${JOB_REPUBLISH_SECONDS:-600}
FAILPOINTS: ${FAILPOINTS:-0} # 1 только на тестовом стенде: файл failpoint-<точка>.png один раз роняет процесс
x-seaweedfs: &seaweedfs
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
restart: unless-stopped
stop_grace_period: 20s
services:
postgres:
image: postgres:18.6-alpine@sha256:10a5c69a83035ab5239f5620d708520e8b28f5aa1f2b20143e6024532edadfb5
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=256MB"]
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 15
deploy:
resources:
limits: { cpus: "${DB_CPUS:-2}", memory: 1g }
stop_grace_period: 30s
restart: unless-stopped
networks: [backend]
# Хранилище SeaweedFS: master + volume + filer, данные в volume s3data. Только во внутренней сети storage:
# его HTTP-интерфейсы (master :9333, filer :8888, volume :8080) не требуют подписи S3 и недоступны даже из API.
seaweedfs:
<<: *seaweedfs
command: ["server", "-dir=/data", "-ip=seaweedfs", "-ip.bind=0.0.0.0", "-filer", "-master.volumeSizeLimitMB=1024"]
volumes: [s3data:/data]
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:8888/"]
interval: 2s
timeout: 2s
retries: 30
start_period: 5s
deploy:
resources:
limits: { cpus: "1", memory: 512m }
networks: [storage]
# S3-шлюз — единственный путь к байтам: проверяет SigV4 и права identities (secrets/s3_identities.json).
# Встроенный IAM API, Iceberg и Lance выключены; порт не опубликован на хост.
s3:
<<: *seaweedfs
# concurrentUploadLimitMB: шлюз буферизует чанки (4 MiB) каждой загрузки; сверх лимита новые загрузки ждут,
# а не выводят шлюз за лимит памяти (эксперимент research/upload_memory.py: при 256m без лимита — OOM).
command: ["s3", "-filer=seaweedfs:8888", "-port=9000", "-config=/run/secrets/s3_identities",
"-iam=false", "-port.iceberg=0", "-port.lance=0", "-concurrentUploadLimitMB=256"]
secrets: [s3_identities]
depends_on:
seaweedfs: { condition: service_healthy }
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:9000/healthz"]
interval: 2s
timeout: 2s
retries: 30
deploy:
resources:
limits: { cpus: "1", memory: 768m }
networks: [storage, backend]
# Одноразовые сервисы до старта реплик: схема БД (advisory lock) и bucket (повторный запуск ничего не меняет).
migrate:
<<: *app
command: ["node", "src/main.ts", "migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
depends_on:
postgres: { condition: service_healthy }
s3-init:
<<: *app
command: ["node", "src/main.ts", "s3-init"]
environment: *s3-env
secrets:
- { source: s3_admin_access_key_id, target: s3_access_key_id }
- { source: s3_admin_secret_access_key, target: s3_secret_access_key }
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
depends_on:
s3: { condition: service_healthy }
# Брокер: RabbitMQ, quorum-очередь на диске (volume rabbitmq-data), publisher confirms, ручной ack.
# hostname фиксирует имя узла rabbit@rabbitmq: данные в volume привязаны к нему и переживают пересоздание.
rabbitmq:
image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5
hostname: rabbitmq
volumes:
- rabbitmq-data:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
secrets: [rabbitmq_definitions]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 30
start_period: 20s
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
networks: [backend]
api:
<<: *app
environment:
<<: [*s3-env, *jobs-env, *otel-env]
OTEL_SERVICE_NAME: messenger-api
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-2500}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
secrets:
- database_url
- { source: broker_url_api, target: broker_url }
- { source: s3_api_access_key_id, target: s3_access_key_id }
- { source: s3_api_secret_access_key, target: s3_secret_access_key }
# rabbitmq не в depends_on: API принимает загрузки и без брокера, задания ждут в outbox.
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 3
# Lab6: старт с OpenTelemetry SDK (загрузка instrumentations через ESM-hook) на занятом хосте — до ~25 с.
# Успешная проверка сразу делает контейнер healthy; start_period только прощает неудачи при старте.
start_period: 60s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-256m}" }
stop_grace_period: 15s
restart: unless-stopped # упавшая реплика поднимается сама; после `docker stop` остаётся остановленной
# Worker: отдельный процесс без HTTP, конкурирующие consumers одной очереди. WORKER_CONCURRENCY = prefetch =
# одновременных заданий на реплику; память ограничена контейнером, libvips — один поток, без кэша.
worker:
<<: *app
command: ["node", "--import", "./src/telemetry.ts", "src/main.ts", "worker"]
environment:
<<: [*s3-env, *jobs-env, *otel-env]
OTEL_SERVICE_NAME: messenger-worker
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
secrets:
- database_url
- { source: broker_url_worker, target: broker_url }
- { source: s3_worker_access_key_id, target: s3_access_key_id }
- { source: s3_worker_secret_access_key, target: s3_secret_access_key }
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
rabbitmq: { condition: service_healthy }
deploy:
replicas: ${WORKER_REPLICAS:-2}
resources:
limits: { cpus: "${WORKER_CPUS:-1}", memory: "${WORKER_MEMORY:-768m}" }
stop_grace_period: 15s # SIGTERM: consumer отменяется, текущие задания до 10 с, затем выход
restart: unless-stopped
# Периодическая сверка БД ↔ S3 с grace period (src/reconcile.ts). Свои S3-ключи: объекты + листинг bucket.
reconciler:
<<: *app
command: ["node", "src/main.ts", "reconcile", "--apply", "--every=${RECONCILE_INTERVAL_SECONDS:-300}"]
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets:
- database_url
- { source: s3_reconciler_access_key_id, target: s3_access_key_id }
- { source: s3_reconciler_secret_access_key, target: s3_secret_access_key }
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
restart: unless-stopped
lb:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
volumes: ["./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # статистика HAProxy для оператора, это не API
depends_on:
api: { condition: service_healthy }
deploy:
resources:
limits: { cpus: "${LB_CPUS:-1}", memory: 128m }
restart: unless-stopped
networks: [frontend, backend]
# ---------- Наблюдаемость (lab6). Конфиги — infra/otel, infra/prometheus, infra/loki, infra/tempo, infra/grafana.
# Collector — единственный компонент telemetry в сети backend: принимает OTLP от api/worker и скрейпит
# RabbitMQ и HAProxy; в хранилища пишет по сети telemetry. Ничего не публикуется на хост, кроме Grafana.
otel-collector:
image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849
volumes: ["./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro"]
environment:
GOMEMLIMIT: 180MiB # мягкий предел GC ниже лимита контейнера; жёсткий отказ приёма — memory_limiter
deploy:
resources:
limits: { cpus: "1", memory: 256m }
restart: unless-stopped
logging: *logging
networks: [backend, telemetry]
prometheus:
image: prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
- --enable-feature=created-timestamp-zero-ingestion # новая серия счётчика начинается с 0: rate() видит первые ошибки
- --query.lookback-delta=1m # данные остановленной реплики перестают учитываться через минуту, а не через 5
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheus-data:/prometheus
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
logging: *logging
networks: [telemetry]
loki:
image: grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
deploy:
resources:
limits: { cpus: "1", memory: 384m }
stop_grace_period: 30s
restart: unless-stopped
logging: *logging
networks: [telemetry]
tempo:
image: grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo-data:/var/tempo
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
logging: *logging
networks: [telemetry]
# Grafana: интерфейс и запросы к хранилищам; datasources и dashboard импортируются из Git (provisioning).
grafana:
image: grafana/grafana:13.2.5@sha256:9f7aa1e1e3246aade5792d35a738049f7f32f89cd51d1956cf426a4e86268a84
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_SERVER_ROOT_URL: http://localhost:${GRAFANA_PORT:-3000}
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_NEWS_NEWS_FEED_ENABLED: "false"
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 30
deploy:
resources:
limits: { cpus: "0.5", memory: 256m }
restart: unless-stopped
logging: *logging
networks: [frontend, telemetry]
networks:
frontend: {}
backend: { internal: true } # api, worker, postgres, rabbitmq и s3-шлюз недоступны с хоста и не имеют выхода наружу
storage: { internal: true } # внутренние компоненты SeaweedFS видит только s3-шлюз
telemetry: { internal: true } # collector → prometheus/loki/tempo ← grafana; приложение сюда не подключено
volumes:
pgdata: {}
s3data: {}
rabbitmq-data: {}
prometheus-data: {}
loki-data: {}
tempo-data: {}
grafana-data: {}
secrets:
database_url: { file: ./secrets/database_url }
postgres_password: { file: ./secrets/postgres_password }
s3_identities: { file: ./secrets/s3_identities.json }
s3_admin_access_key_id: { file: ./secrets/s3_admin_access_key_id }
s3_admin_secret_access_key: { file: ./secrets/s3_admin_secret_access_key }
s3_api_access_key_id: { file: ./secrets/s3_api_access_key_id }
s3_api_secret_access_key: { file: ./secrets/s3_api_secret_access_key }
s3_reconciler_access_key_id: { file: ./secrets/s3_reconciler_access_key_id }
s3_reconciler_secret_access_key: { file: ./secrets/s3_reconciler_secret_access_key }
s3_worker_access_key_id: { file: ./secrets/s3_worker_access_key_id }
s3_worker_secret_access_key: { file: ./secrets/s3_worker_secret_access_key }
rabbitmq_definitions: { file: ./secrets/rabbitmq_definitions.json }
broker_url_api: { file: ./secrets/broker_url_api }
broker_url_worker: { file: ./secrets/broker_url_worker }
grafana_admin_password: { file: ./secrets/grafana_admin_password }