Files
mtusi2026lab6/compose.yaml
T

477 lines
19 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Lab5: клиент → HAProxy → API × N → PostgreSQL (данные, метаданные, jobs, outbox), Redis (сессии, кэш),
# приватный S3 (MinIO: staging s/, оригиналы o/, варианты v/).
# relay: outbox → RabbitMQ (publisher confirms); worker: RabbitMQ → варианты → S3 + PostgreSQL.
# Lab6: API / relay / worker —OTLP/HTTP→ otel-collector → Tempo (traces), Loki (logs), Prometheus (metrics,
# pull :8889, alert rules) → Grafana (provisioning из infra/grafana). Сеть telemetry отделена от back.
# Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait
name: mtusi2026lab6
x-api-env: &api-env
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-redis}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_DELAY: ${DRAIN_DELAY:-3s}
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-8s}
DB_POOL_MAX: ${DB_POOL_MAX:-20}
CACHE_INVALIDATION: ${CACHE_INVALIDATION:-version}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
WEBP_QUALITY: ${WEBP_QUALITY:-80}
WORKER_PIXEL_BUDGET: ${WORKER_PIXEL_BUDGET:-60000000}
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2s}
RETRY_MAX_DELAY: ${RETRY_MAX_DELAY:-60s}
# OpenTelemetry SDK (стандартные переменные; service.name задаёт каждый сервис)
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_always_on}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
services:
lb:
image: haproxy:3.0.5-alpine
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
environment:
LB_BALANCE: ${LB_BALANCE:-roundrobin}
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
restart: unless-stopped
networks: [front]
deploy:
resources:
limits: { cpus: "${LB_CPUS:-0.50}", memory: 128M }
api:
build: .
image: mtusi2026lab6-api:local
# Без container_name и без ports: реплики доступны только через lb.
environment:
<<: *api-env
OTEL_SERVICE_NAME: messenger-api
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/health/ready"]
interval: 5s
timeout: 2s
retries: 3
start_period: 5s
# DRAIN_DELAY + SHUTDOWN_TIMEOUT < stop_grace_period, иначе SIGKILL оборвёт запросы.
stop_grace_period: 15s
restart: unless-stopped
networks: [front, back, telemetry]
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "${API_CPUS:-0.50}", memory: "${API_MEMORY:-256M}" }
# Однократные миграции до старта API: реплики не гоняются за схему.
migrate:
image: migrate/migrate:v4.17.0
volumes:
- ./migrations:/migrations:ro
secrets: [database_url]
entrypoint: ["/bin/sh", "-c", "exec migrate -path /migrations -database \"$$(cat /run/secrets/database_url)\" up"]
depends_on:
postgres:
condition: service_healthy
restart: "no"
networks: [back]
postgres:
image: postgres:16.4-alpine
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
secrets: [postgres_password]
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "${PG_CPUS:-1.00}", memory: "${PG_MEMORY:-512M}" }
redis:
image: redis:7.4.1-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "${REDIS_CPUS:-0.50}", memory: 256M }
# S3-совместимое хранилище байтов вложений: MinIO RELEASE.2025-04-22T22-12-26Z.
# Официальные образы minio/minio убраны из Docker Hub и quay.io (проверено в CI),
# поэтому используется сборка Bitnami того же релиза, закреплённая по digest.
# Порты не публикуются, консоль выключена (MINIO_BROWSER=off): доступ только из сети back.
s3:
image: bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01
entrypoint: ["/bin/bash", "-c"]
command:
- export MINIO_ROOT_USER="$$(cat /run/secrets/s3_root_user)" MINIO_ROOT_PASSWORD="$$(cat /run/secrets/s3_root_password)"; exec /opt/bitnami/minio/bin/minio server /bitnami/minio/data --address :9000
environment:
MINIO_BROWSER: "off"
secrets: [s3_root_user, s3_root_password]
volumes:
- s3_data:/bitnami/minio/data
healthcheck:
# Без curl в образе: HTTP/1.1-запрос к /minio/health/live через /dev/tcp и read (только bash).
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9000 && printf 'GET /minio/health/live HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && read -r line <&3 && [[ $$line == *' 200 '* ]]"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "${S3_CPUS:-0.50}", memory: 512M }
# Идемпотентный bootstrap: приватный bucket, политика и отдельный пользователь
# API с правами только на этот bucket. Root-учётка есть только у s3 и s3-init.
s3-init:
image: bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1
entrypoint: ["/bin/sh", "/init/init.sh"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
PATH: /opt/bitnami/minio-client/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
MC_CONFIG_DIR: /tmp/.mc
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
volumes:
- ./infra/s3:/init:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
networks: [back]
# Сверка PostgreSQL и S3: брошенные загрузки, сироты, потерянные объекты.
# Безопасный grace period 1h; одновременно работает один экземпляр (advisory lock).
reconciler:
image: mtusi2026lab6-api:local
entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"]
environment:
<<: *api-env
CACHE_URL_FILE: "" # Redis reconciler не нужен
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
# HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что
# процесс reconcile жив (нужна и для docker compose up --wait).
healthcheck:
test: ["CMD-SHELL", "pgrep -x reconcile >/dev/null || pidof reconcile >/dev/null"]
interval: 10s
timeout: 3s
retries: 3
start_period: 5s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
networks: [back]
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
# Broker: quorum-очереди на диске (volume), без management UI, порты не публикуются.
# Пользователь и пароль — из secret rabbitmq.conf (guest отключён вне localhost).
rabbitmq:
image: rabbitmq:4.0.9-alpine
hostname: rabbitmq # имя узла фиксировано: данные в volume привязаны к нему
secrets:
- source: rabbitmq_conf
target: /etc/rabbitmq/conf.d/20-credentials.conf
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 20
start_period: 10s
restart: unless-stopped
# telemetry — только чтобы Prometheus забирал :15692 (метрики очередей, без AMQP-учётки).
networks: [back, telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 512M }
# Relay: публикует transactional outbox с publisher confirms, возвращает в очередь
# работу с истёкшим lease, чистит staging. Не зависит от доступности API.
relay:
image: mtusi2026lab6-api:local
entrypoint: ["/app/pipeline", "relay"]
environment:
<<: *api-env
CACHE_URL_FILE: ""
BROKER_URL_FILE: /run/secrets/broker_url
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
OTEL_SERVICE_NAME: messenger-relay
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
healthcheck:
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
interval: 10s
timeout: 3s
retries: 3
start_period: 5s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
restart: unless-stopped
networks: [back, telemetry]
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
# Worker: отдельный процесс; prefetch = WORKER_CONCURRENCY, lease = JOB_TIMEOUT_SECONDS,
# память ограничена контейнером и пиксельным бюджетом. SIGTERM → дорабатывает текущие задания.
worker:
image: mtusi2026lab6-api:local
entrypoint: ["/app/pipeline", "worker"]
environment:
<<: *api-env
CACHE_URL_FILE: ""
BROKER_URL_FILE: /run/secrets/broker_url
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
OTEL_SERVICE_NAME: messenger-worker
# Мягкий лимит кучи Go ≈ 75% лимита контейнера: без него GC (GOGC=100) даёт куче
# расти до ~2× живого объёма, и пик подходил вплотную к 768M (замер прогона 4).
GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-560MiB}
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
healthcheck:
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
interval: 10s
timeout: 3s
retries: 3
start_period: 5s
stop_grace_period: 70s
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
restart: unless-stopped
networks: [back, telemetry]
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits: { cpus: "${WORKER_CPUS:-1.00}", memory: "${WORKER_MEMORY:-768M}" }
# ---------------- Наблюдаемость (lab6) ----------------
# Collector: приём OTLP (4317 gRPC / 4318 HTTP) только из сети telemetry, порты
# наружу не публикуются. memory_limiter + ограниченные очереди экспортёров:
# отказ Tempo/Loki не приводит к росту памяти (infra/otel/collector.yaml).
otel-collector:
image: otel/opentelemetry-collector-contrib:0.116.1
command: ["--config=/etc/otelcol/${COLLECTOR_CONFIG:-collector.yaml}"]
volumes:
- ./infra/otel:/etc/otelcol:ro
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "${COLLECTOR_CPUS:-0.50}", memory: 256M }
# Метрики: TSDB на volume (7 дней), alert rules, exemplars → Tempo.
# UI/API только на localhost хоста (администрирование, проверки).
prometheus:
image: prom/prometheus:v3.1.0
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --enable-feature=exemplar-storage
- --web.enable-lifecycle
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheus_data:/prometheus
ports:
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 512M }
# Логи: Loki (OTLP /otlp), файловое хранилище на volume, retention 7 дней.
loki:
image: grafana/loki:3.3.2
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3100/ready"]
interval: 5s
timeout: 3s
retries: 30
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5)
# Трассы: Tempo, локальный backend на volume, retention 7 дней.
# tempo-init один раз выдаёт volume пользователю tempo (uid 10001).
tempo-init:
image: &tempo-image grafana/tempo:2.6.1
user: root
entrypoint: ["chown", "10001:10001", "/var/tempo"]
volumes:
- tempo_data:/var/tempo
restart: "no"
networks: [telemetry]
tempo:
image: *tempo-image
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
depends_on:
tempo-init:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3200/ready"]
interval: 5s
timeout: 3s
retries: 30
start_period: 10s
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5)
# Grafana: только интерфейс и запросы; datasources и dashboard — provisioning из Git.
# Пароль admin — secret; порт только на localhost.
grafana:
image: grafana/grafana:11.4.0
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_FEATURE_TOGGLES_ENABLE: traceqlEditor
# Только для снимка dashboard в CI (GRAFANA_ANONYMOUS=true): анонимный Viewer.
GF_AUTH_ANONYMOUS_ENABLED: ${GRAFANA_ANONYMOUS:-false}
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
depends_on:
prometheus:
condition: service_healthy
loki:
condition: service_healthy
tempo:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
interval: 5s
timeout: 3s
retries: 30
restart: unless-stopped
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.50", memory: 256M }
networks:
# front: lb ↔ api; back: api ↔ хранилища и broker. LB не видит БД, S3 и broker.
# telemetry: процессы приложения → Collector → хранилища сигналов → Grafana;
# backend-ы телеметрии не видят БД, S3 и broker (кроме порта метрик RabbitMQ).
front:
back:
telemetry:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
rabbitmq_conf:
file: ./secrets/rabbitmq.conf
broker_url:
file: ./secrets/broker_url
grafana_admin_password:
file: ./secrets/grafana_admin_password
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
rabbitmq_data:
s3_data:
pg_data:
redis_data: