# Lab5: клиент → HAProxy → API × N → PostgreSQL (данные, метаданные, jobs, outbox), Redis (сессии, кэш), # приватный S3 (MinIO: staging s/, оригиналы o/, варианты v/). # relay: outbox → RabbitMQ (publisher confirms); worker: RabbitMQ → варианты → S3 + PostgreSQL. # Lab6: API / relay / worker —OTLP/HTTP→ otel-collector → Tempo (traces), Loki (logs), Prometheus (metrics, # pull :8889, alert rules) → Grafana (provisioning из infra/grafana). Сеть telemetry отделена от back. # Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait name: mtusi2026lab6 x-api-env: &api-env APP_HOST: ${APP_HOST:-0.0.0.0} APP_PORT: ${APP_PORT:-8080} APP_ENV: ${APP_ENV:-lab} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} SESSION_STORE: ${SESSION_STORE:-redis} DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} DRAIN_DELAY: ${DRAIN_DELAY:-3s} SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-8s} DB_POOL_MAX: ${DB_POOL_MAX:-20} CACHE_INVALIDATION: ${CACHE_INVALIDATION:-version} S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} PIPELINE_VERSION: ${PIPELINE_VERSION:-1} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} WEBP_QUALITY: ${WEBP_QUALITY:-80} WORKER_PIXEL_BUDGET: ${WORKER_PIXEL_BUDGET:-60000000} RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2s} RETRY_MAX_DELAY: ${RETRY_MAX_DELAY:-60s} # OpenTelemetry SDK (стандартные переменные; service.name задаёт каждый сервис) OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_always_on} OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} services: lb: image: haproxy:3.0.5-alpine ports: - "127.0.0.1:${LB_PORT:-8080}:8080" - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" environment: LB_BALANCE: ${LB_BALANCE:-roundrobin} volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro depends_on: api: condition: service_healthy restart: unless-stopped networks: [front] deploy: resources: limits: { cpus: "${LB_CPUS:-0.50}", memory: 128M } api: build: . image: mtusi2026lab6-api:local # Без container_name и без ports: реплики доступны только через lb. environment: <<: *api-env OTEL_SERVICE_NAME: messenger-api secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] depends_on: migrate: condition: service_completed_successfully s3-init: condition: service_completed_successfully postgres: condition: service_healthy redis: condition: service_healthy healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/health/ready"] interval: 5s timeout: 2s retries: 3 start_period: 5s # DRAIN_DELAY + SHUTDOWN_TIMEOUT < stop_grace_period, иначе SIGKILL оборвёт запросы. stop_grace_period: 15s restart: unless-stopped networks: [front, back, telemetry] deploy: replicas: ${API_REPLICAS:-2} resources: limits: { cpus: "${API_CPUS:-0.50}", memory: "${API_MEMORY:-256M}" } # Однократные миграции до старта API: реплики не гоняются за схему. migrate: image: migrate/migrate:v4.17.0 volumes: - ./migrations:/migrations:ro secrets: [database_url] entrypoint: ["/bin/sh", "-c", "exec migrate -path /migrations -database \"$$(cat /run/secrets/database_url)\" up"] depends_on: postgres: condition: service_healthy restart: "no" networks: [back] postgres: image: postgres:16.4-alpine command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] environment: POSTGRES_USER: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password POSTGRES_DB: messenger secrets: [postgres_password] volumes: - pg_data:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] interval: 5s timeout: 3s retries: 5 restart: unless-stopped networks: [back] deploy: resources: limits: { cpus: "${PG_CPUS:-1.00}", memory: "${PG_MEMORY:-512M}" } redis: image: redis:7.4.1-alpine command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] volumes: - redis_data:/data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s timeout: 3s retries: 5 restart: unless-stopped networks: [back] deploy: resources: limits: { cpus: "${REDIS_CPUS:-0.50}", memory: 256M } # S3-совместимое хранилище байтов вложений: MinIO RELEASE.2025-04-22T22-12-26Z. # Официальные образы minio/minio убраны из Docker Hub и quay.io (проверено в CI), # поэтому используется сборка Bitnami того же релиза, закреплённая по digest. # Порты не публикуются, консоль выключена (MINIO_BROWSER=off): доступ только из сети back. s3: image: bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01 entrypoint: ["/bin/bash", "-c"] command: - export MINIO_ROOT_USER="$$(cat /run/secrets/s3_root_user)" MINIO_ROOT_PASSWORD="$$(cat /run/secrets/s3_root_password)"; exec /opt/bitnami/minio/bin/minio server /bitnami/minio/data --address :9000 environment: MINIO_BROWSER: "off" secrets: [s3_root_user, s3_root_password] volumes: - s3_data:/bitnami/minio/data healthcheck: # Без curl в образе: HTTP/1.1-запрос к /minio/health/live через /dev/tcp и read (только bash). test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9000 && printf 'GET /minio/health/live HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && read -r line <&3 && [[ $$line == *' 200 '* ]]"] interval: 5s timeout: 3s retries: 20 restart: unless-stopped networks: [back] deploy: resources: limits: { cpus: "${S3_CPUS:-0.50}", memory: 512M } # Идемпотентный bootstrap: приватный bucket, политика и отдельный пользователь # API с правами только на этот bucket. Root-учётка есть только у s3 и s3-init. s3-init: image: bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1 entrypoint: ["/bin/sh", "/init/init.sh"] environment: S3_BUCKET: ${S3_BUCKET:-messenger-attachments} PATH: /opt/bitnami/minio-client/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin MC_CONFIG_DIR: /tmp/.mc secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] volumes: - ./infra/s3:/init:ro depends_on: s3: condition: service_healthy restart: "no" networks: [back] # Сверка PostgreSQL и S3: брошенные загрузки, сироты, потерянные объекты. # Безопасный grace period 1h; одновременно работает один экземпляр (advisory lock). reconciler: image: mtusi2026lab6-api:local entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"] environment: <<: *api-env CACHE_URL_FILE: "" # Redis reconciler не нужен secrets: [database_url, s3_access_key_id, s3_secret_access_key] # HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что # процесс reconcile жив (нужна и для docker compose up --wait). healthcheck: test: ["CMD-SHELL", "pgrep -x reconcile >/dev/null || pidof reconcile >/dev/null"] interval: 10s timeout: 3s retries: 3 start_period: 5s depends_on: migrate: condition: service_completed_successfully s3-init: condition: service_completed_successfully restart: unless-stopped networks: [back] deploy: resources: limits: { cpus: "0.25", memory: 128M } # Broker: quorum-очереди на диске (volume), без management UI, порты не публикуются. # Пользователь и пароль — из secret rabbitmq.conf (guest отключён вне localhost). rabbitmq: image: rabbitmq:4.0.9-alpine hostname: rabbitmq # имя узла фиксировано: данные в volume привязаны к нему secrets: - source: rabbitmq_conf target: /etc/rabbitmq/conf.d/20-credentials.conf volumes: - rabbitmq_data:/var/lib/rabbitmq healthcheck: test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] interval: 5s timeout: 10s retries: 20 start_period: 10s restart: unless-stopped # telemetry — только чтобы Prometheus забирал :15692 (метрики очередей, без AMQP-учётки). networks: [back, telemetry] deploy: resources: limits: { cpus: "0.50", memory: 512M } # Relay: публикует transactional outbox с publisher confirms, возвращает в очередь # работу с истёкшим lease, чистит staging. Не зависит от доступности API. relay: image: mtusi2026lab6-api:local entrypoint: ["/app/pipeline", "relay"] environment: <<: *api-env CACHE_URL_FILE: "" BROKER_URL_FILE: /run/secrets/broker_url FAULT_POINTS: ${RELAY_FAULT_POINTS:-} OTEL_SERVICE_NAME: messenger-relay secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url] healthcheck: test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"] interval: 10s timeout: 3s retries: 3 start_period: 5s depends_on: migrate: condition: service_completed_successfully s3-init: condition: service_completed_successfully rabbitmq: condition: service_healthy restart: unless-stopped networks: [back, telemetry] deploy: resources: limits: { cpus: "0.25", memory: 128M } # Worker: отдельный процесс; prefetch = WORKER_CONCURRENCY, lease = JOB_TIMEOUT_SECONDS, # память ограничена контейнером и пиксельным бюджетом. SIGTERM → дорабатывает текущие задания. worker: image: mtusi2026lab6-api:local entrypoint: ["/app/pipeline", "worker"] environment: <<: *api-env CACHE_URL_FILE: "" BROKER_URL_FILE: /run/secrets/broker_url FAULT_POINTS: ${WORKER_FAULT_POINTS:-} OTEL_SERVICE_NAME: messenger-worker # Мягкий лимит кучи Go ≈ 75% лимита контейнера: без него GC (GOGC=100) даёт куче # расти до ~2× живого объёма, и пик подходил вплотную к 768M (замер прогона 4). GOMEMLIMIT: ${WORKER_GOMEMLIMIT:-560MiB} secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url] healthcheck: test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"] interval: 10s timeout: 3s retries: 3 start_period: 5s stop_grace_period: 70s depends_on: migrate: condition: service_completed_successfully s3-init: condition: service_completed_successfully rabbitmq: condition: service_healthy restart: unless-stopped networks: [back, telemetry] deploy: replicas: ${WORKER_REPLICAS:-1} resources: limits: { cpus: "${WORKER_CPUS:-1.00}", memory: "${WORKER_MEMORY:-768M}" } # ---------------- Наблюдаемость (lab6) ---------------- # Collector: приём OTLP (4317 gRPC / 4318 HTTP) только из сети telemetry, порты # наружу не публикуются. memory_limiter + ограниченные очереди экспортёров: # отказ Tempo/Loki не приводит к росту памяти (infra/otel/collector.yaml). otel-collector: image: otel/opentelemetry-collector-contrib:0.116.1 command: ["--config=/etc/otelcol/${COLLECTOR_CONFIG:-collector.yaml}"] volumes: - ./infra/otel:/etc/otelcol:ro restart: unless-stopped networks: [telemetry] deploy: resources: limits: { cpus: "${COLLECTOR_CPUS:-0.50}", memory: 256M } # Метрики: TSDB на volume (7 дней), alert rules, exemplars → Tempo. # UI/API только на localhost хоста (администрирование, проверки). prometheus: image: prom/prometheus:v3.1.0 command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=7d - --enable-feature=exemplar-storage - --web.enable-lifecycle volumes: - ./infra/prometheus:/etc/prometheus:ro - prometheus_data:/prometheus ports: - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"] interval: 5s timeout: 3s retries: 20 restart: unless-stopped networks: [telemetry] deploy: resources: limits: { cpus: "0.50", memory: 512M } # Логи: Loki (OTLP /otlp), файловое хранилище на volume, retention 7 дней. loki: image: grafana/loki:3.3.2 command: ["-config.file=/etc/loki/loki.yaml"] volumes: - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro - loki_data:/loki healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:3100/ready"] interval: 5s timeout: 3s retries: 30 restart: unless-stopped networks: [telemetry] deploy: resources: limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5) # Трассы: Tempo, локальный backend на volume, retention 7 дней. # tempo-init один раз выдаёт volume пользователю tempo (uid 10001). tempo-init: image: &tempo-image grafana/tempo:2.6.1 user: root entrypoint: ["chown", "10001:10001", "/var/tempo"] volumes: - tempo_data:/var/tempo restart: "no" networks: [telemetry] tempo: image: *tempo-image command: ["-config.file=/etc/tempo/tempo.yaml"] volumes: - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo_data:/var/tempo depends_on: tempo-init: condition: service_completed_successfully healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:3200/ready"] interval: 5s timeout: 3s retries: 30 start_period: 10s restart: unless-stopped networks: [telemetry] deploy: resources: limits: { cpus: "0.50", memory: 1G } # 512M: OOM под нагрузкой трека 1 (~14 тыс. spans на 1000 запросов, прогон 5) # Grafana: только интерфейс и запросы; datasources и dashboard — provisioning из Git. # Пароль admin — secret; порт только на localhost. grafana: image: grafana/grafana:11.4.0 environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password GF_USERS_ALLOW_SIGN_UP: "false" GF_ANALYTICS_REPORTING_ENABLED: "false" GF_ANALYTICS_CHECK_FOR_UPDATES: "false" GF_FEATURE_TOGGLES_ENABLE: traceqlEditor # Только для снимка dashboard в CI (GRAFANA_ANONYMOUS=true): анонимный Viewer. GF_AUTH_ANONYMOUS_ENABLED: ${GRAFANA_ANONYMOUS:-false} GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer secrets: [grafana_admin_password] volumes: - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro - grafana_data:/var/lib/grafana ports: - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" depends_on: prometheus: condition: service_healthy loki: condition: service_healthy tempo: condition: service_healthy healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"] interval: 5s timeout: 3s retries: 30 restart: unless-stopped networks: [telemetry] deploy: resources: limits: { cpus: "0.50", memory: 256M } networks: # front: lb ↔ api; back: api ↔ хранилища и broker. LB не видит БД, S3 и broker. # telemetry: процессы приложения → Collector → хранилища сигналов → Grafana; # backend-ы телеметрии не видят БД, S3 и broker (кроме порта метрик RabbitMQ). front: back: telemetry: secrets: postgres_password: file: ./secrets/postgres_password database_url: file: ./secrets/database_url cache_url: file: ./secrets/cache_url s3_root_user: file: ./secrets/s3_root_user s3_root_password: file: ./secrets/s3_root_password s3_access_key_id: file: ./secrets/s3_access_key_id s3_secret_access_key: file: ./secrets/s3_secret_access_key rabbitmq_conf: file: ./secrets/rabbitmq.conf broker_url: file: ./secrets/broker_url grafana_admin_password: file: ./secrets/grafana_admin_password volumes: prometheus_data: loki_data: tempo_data: grafana_data: rabbitmq_data: s3_data: pg_data: redis_data: