322 lines
12 KiB
YAML
322 lines
12 KiB
YAML
# Lab5: клиент → HAProxy → API × N → PostgreSQL (данные, метаданные, jobs, outbox), Redis (сессии, кэш),
|
||
# приватный S3 (MinIO: staging s/, оригиналы o/, варианты v/).
|
||
# relay: outbox → RabbitMQ (publisher confirms); worker: RabbitMQ → варианты → S3 + PostgreSQL.
|
||
# Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait
|
||
name: mtusi2026lab5
|
||
|
||
x-api-env: &api-env
|
||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||
APP_PORT: ${APP_PORT:-8080}
|
||
APP_ENV: ${APP_ENV:-lab}
|
||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||
SESSION_STORE: ${SESSION_STORE:-redis}
|
||
DATABASE_URL_FILE: /run/secrets/database_url
|
||
CACHE_URL_FILE: /run/secrets/cache_url
|
||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||
DRAIN_DELAY: ${DRAIN_DELAY:-3s}
|
||
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-8s}
|
||
DB_POOL_MAX: ${DB_POOL_MAX:-20}
|
||
CACHE_INVALIDATION: ${CACHE_INVALIDATION:-version}
|
||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||
S3_REGION: ${S3_REGION:-us-east-1}
|
||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||
WEBP_QUALITY: ${WEBP_QUALITY:-80}
|
||
WORKER_PIXEL_BUDGET: ${WORKER_PIXEL_BUDGET:-60000000}
|
||
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2s}
|
||
RETRY_MAX_DELAY: ${RETRY_MAX_DELAY:-60s}
|
||
|
||
services:
|
||
lb:
|
||
image: haproxy:3.0.5-alpine
|
||
ports:
|
||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||
environment:
|
||
LB_BALANCE: ${LB_BALANCE:-roundrobin}
|
||
volumes:
|
||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||
depends_on:
|
||
api:
|
||
condition: service_healthy
|
||
restart: unless-stopped
|
||
networks: [front]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "${LB_CPUS:-0.50}", memory: 128M }
|
||
|
||
api:
|
||
build: .
|
||
image: mtusi2026lab5-api:local
|
||
# Без container_name и без ports: реплики доступны только через lb.
|
||
environment: *api-env
|
||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
condition: service_completed_successfully
|
||
postgres:
|
||
condition: service_healthy
|
||
redis:
|
||
condition: service_healthy
|
||
healthcheck:
|
||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/health/ready"]
|
||
interval: 5s
|
||
timeout: 2s
|
||
retries: 3
|
||
start_period: 5s
|
||
# DRAIN_DELAY + SHUTDOWN_TIMEOUT < stop_grace_period, иначе SIGKILL оборвёт запросы.
|
||
stop_grace_period: 15s
|
||
restart: unless-stopped
|
||
networks: [front, back]
|
||
deploy:
|
||
replicas: ${API_REPLICAS:-2}
|
||
resources:
|
||
limits: { cpus: "${API_CPUS:-0.50}", memory: "${API_MEMORY:-256M}" }
|
||
|
||
# Однократные миграции до старта API: реплики не гоняются за схему.
|
||
migrate:
|
||
image: migrate/migrate:v4.17.0
|
||
volumes:
|
||
- ./migrations:/migrations:ro
|
||
secrets: [database_url]
|
||
entrypoint: ["/bin/sh", "-c", "exec migrate -path /migrations -database \"$$(cat /run/secrets/database_url)\" up"]
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
restart: "no"
|
||
networks: [back]
|
||
|
||
postgres:
|
||
image: postgres:16.4-alpine
|
||
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
|
||
environment:
|
||
POSTGRES_USER: messenger
|
||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||
POSTGRES_DB: messenger
|
||
secrets: [postgres_password]
|
||
volumes:
|
||
- pg_data:/var/lib/postgresql/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 5
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "${PG_CPUS:-1.00}", memory: "${PG_MEMORY:-512M}" }
|
||
|
||
redis:
|
||
image: redis:7.4.1-alpine
|
||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
||
volumes:
|
||
- redis_data:/data
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 5
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "${REDIS_CPUS:-0.50}", memory: 256M }
|
||
|
||
# S3-совместимое хранилище байтов вложений: MinIO RELEASE.2025-04-22T22-12-26Z.
|
||
# Официальные образы minio/minio убраны из Docker Hub и quay.io (проверено в CI),
|
||
# поэтому используется сборка Bitnami того же релиза, закреплённая по digest.
|
||
# Порты не публикуются, консоль выключена (MINIO_BROWSER=off): доступ только из сети back.
|
||
s3:
|
||
image: bitnamilegacy/minio:2025.4.22@sha256:50cec18ac4184af4671a78aedd5554942c8ae105d51a465fa82037949046da01
|
||
entrypoint: ["/bin/bash", "-c"]
|
||
command:
|
||
- export MINIO_ROOT_USER="$$(cat /run/secrets/s3_root_user)" MINIO_ROOT_PASSWORD="$$(cat /run/secrets/s3_root_password)"; exec /opt/bitnami/minio/bin/minio server /bitnami/minio/data --address :9000
|
||
environment:
|
||
MINIO_BROWSER: "off"
|
||
secrets: [s3_root_user, s3_root_password]
|
||
volumes:
|
||
- s3_data:/bitnami/minio/data
|
||
healthcheck:
|
||
# Без curl в образе: HTTP/1.1-запрос к /minio/health/live через /dev/tcp и read (только bash).
|
||
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9000 && printf 'GET /minio/health/live HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n' >&3 && read -r line <&3 && [[ $$line == *' 200 '* ]]"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 20
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "${S3_CPUS:-0.50}", memory: 512M }
|
||
|
||
# Идемпотентный bootstrap: приватный bucket, политика и отдельный пользователь
|
||
# API с правами только на этот bucket. Root-учётка есть только у s3 и s3-init.
|
||
s3-init:
|
||
image: bitnamilegacy/minio-client:2025.4.16@sha256:8a86e441decf053093c5977d49290298295f850c7843652f93e970ab15871dd1
|
||
entrypoint: ["/bin/sh", "/init/init.sh"]
|
||
environment:
|
||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||
PATH: /opt/bitnami/minio-client/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||
MC_CONFIG_DIR: /tmp/.mc
|
||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||
volumes:
|
||
- ./infra/s3:/init:ro
|
||
depends_on:
|
||
s3:
|
||
condition: service_healthy
|
||
restart: "no"
|
||
networks: [back]
|
||
|
||
# Сверка PostgreSQL и S3: брошенные загрузки, сироты, потерянные объекты.
|
||
# Безопасный grace period 1h; одновременно работает один экземпляр (advisory lock).
|
||
reconciler:
|
||
image: mtusi2026lab5-api:local
|
||
entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"]
|
||
environment:
|
||
<<: *api-env
|
||
CACHE_URL_FILE: "" # Redis reconciler не нужен
|
||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||
# HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что
|
||
# процесс reconcile жив (нужна и для docker compose up --wait).
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pgrep -x reconcile >/dev/null || pidof reconcile >/dev/null"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 5s
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
condition: service_completed_successfully
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "0.25", memory: 128M }
|
||
|
||
# Broker: quorum-очереди на диске (volume), без management UI, порты не публикуются.
|
||
# Пользователь и пароль — из secret rabbitmq.conf (guest отключён вне localhost).
|
||
rabbitmq:
|
||
image: rabbitmq:4.0.9-alpine
|
||
hostname: rabbitmq # имя узла фиксировано: данные в volume привязаны к нему
|
||
secrets:
|
||
- source: rabbitmq_conf
|
||
target: /etc/rabbitmq/conf.d/20-credentials.conf
|
||
volumes:
|
||
- rabbitmq_data:/var/lib/rabbitmq
|
||
healthcheck:
|
||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||
interval: 5s
|
||
timeout: 10s
|
||
retries: 20
|
||
start_period: 10s
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "0.50", memory: 512M }
|
||
|
||
# Relay: публикует transactional outbox с publisher confirms, возвращает в очередь
|
||
# работу с истёкшим lease, чистит staging. Не зависит от доступности API.
|
||
relay:
|
||
image: mtusi2026lab5-api:local
|
||
entrypoint: ["/app/pipeline", "relay"]
|
||
environment:
|
||
<<: *api-env
|
||
CACHE_URL_FILE: ""
|
||
BROKER_URL_FILE: /run/secrets/broker_url
|
||
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
|
||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 5s
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
condition: service_completed_successfully
|
||
rabbitmq:
|
||
condition: service_healthy
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
resources:
|
||
limits: { cpus: "0.25", memory: 128M }
|
||
|
||
# Worker: отдельный процесс; prefetch = WORKER_CONCURRENCY, lease = JOB_TIMEOUT_SECONDS,
|
||
# память ограничена контейнером и пиксельным бюджетом. SIGTERM → дорабатывает текущие задания.
|
||
worker:
|
||
image: mtusi2026lab5-api:local
|
||
entrypoint: ["/app/pipeline", "worker"]
|
||
environment:
|
||
<<: *api-env
|
||
CACHE_URL_FILE: ""
|
||
BROKER_URL_FILE: /run/secrets/broker_url
|
||
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
|
||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pgrep -x pipeline >/dev/null || pidof pipeline >/dev/null"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 5s
|
||
stop_grace_period: 70s
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
condition: service_completed_successfully
|
||
rabbitmq:
|
||
condition: service_healthy
|
||
restart: unless-stopped
|
||
networks: [back]
|
||
deploy:
|
||
replicas: ${WORKER_REPLICAS:-1}
|
||
resources:
|
||
limits: { cpus: "${WORKER_CPUS:-1.00}", memory: "${WORKER_MEMORY:-768M}" }
|
||
|
||
networks:
|
||
# front: lb ↔ api; back: api ↔ хранилища и broker. LB не видит БД, S3 и broker.
|
||
front:
|
||
back:
|
||
|
||
secrets:
|
||
postgres_password:
|
||
file: ./secrets/postgres_password
|
||
database_url:
|
||
file: ./secrets/database_url
|
||
cache_url:
|
||
file: ./secrets/cache_url
|
||
s3_root_user:
|
||
file: ./secrets/s3_root_user
|
||
s3_root_password:
|
||
file: ./secrets/s3_root_password
|
||
s3_access_key_id:
|
||
file: ./secrets/s3_access_key_id
|
||
s3_secret_access_key:
|
||
file: ./secrets/s3_secret_access_key
|
||
rabbitmq_conf:
|
||
file: ./secrets/rabbitmq.conf
|
||
broker_url:
|
||
file: ./secrets/broker_url
|
||
|
||
volumes:
|
||
rabbitmq_data:
|
||
s3_data:
|
||
pg_data:
|
||
redis_data:
|