Files
mtusi2026lab4-fork/compose.yaml
T

397 lines
14 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Лабораторная 4: HAProxy -> API x N (по умолчанию 2) -> PostgreSQL (метаданные) + Redis (кэш сессий)
# + SeaweedFS S3 (приватный бакет с байтами вложений).
# Сети: backend — lb, api, postgres, redis; storage (internal) — api, s3, s3-init.
# LB не видит S3, S3 не видит PostgreSQL; наружу (127.0.0.1) опубликован только LB.
name: messenger-lab4
x-api-env: &api-env
APP_ENV: ${APP_ENV:-lab}
APP_VERSION: ${APP_VERSION:-lab4}
SESSION_STORE: postgres
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
DB_POOL_MIN: ${DB_POOL_MIN:-1}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-3}
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.25}
CACHE_RETRY_AFTER_SECONDS: ${CACHE_RETRY_AFTER_SECONDS:-5}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
READY_DB_TIMEOUT_SECONDS: ${READY_DB_TIMEOUT_SECONDS:-1}
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
GRACEFUL_TIMEOUT_SECONDS: ${GRACEFUL_TIMEOUT_SECONDS:-10}
KEEP_ALIVE_SECONDS: ${KEEP_ALIVE_SECONDS:-30}
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4}
# Реплики НЕ применяют миграции — это делает сервис migrate.
RUN_MIGRATIONS_ON_START: "false"
# lab4: S3 (credentials API — отдельные, только Read/Write/List одного бакета)
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10}
S3_MAX_ATTEMPTS: ${S3_MAX_ATTEMPTS:-2}
# Общий бюджет вызова S3 (вкл. DNS) и circuit breaker — см. src/main.py s3_call
S3_DOWNLOAD_DEADLINE_SECONDS: ${S3_DOWNLOAD_DEADLINE_SECONDS:-5}
S3_UPLOAD_DEADLINE_SECONDS: ${S3_UPLOAD_DEADLINE_SECONDS:-10}
S3_CIRCUIT_OPEN_SECONDS: ${S3_CIRCUIT_OPEN_SECONDS:-5}
# Лимиты загрузки (не выше контракта: 10 MiB, 40 Мп)
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-4}
UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5}
UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-60}
IMAGE_DECODE_CONCURRENCY: ${IMAGE_DECODE_CONCURRENCY:-2}
# Сборщик незавершённых загрузок и объектов-сирот (src/reconcile.py)
ATTACHMENT_GC_INTERVAL_SECONDS: ${ATTACHMENT_GC_INTERVAL_SECONDS:-300}
ATTACHMENT_GC_GRACE_SECONDS: ${ATTACHMENT_GC_GRACE_SECONDS:-900}
ORPHAN_SWEEP_INTERVAL_SECONDS: ${ORPHAN_SWEEP_INTERVAL_SECONDS:-3600}
x-api-healthcheck: &api-healthcheck
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
interval: 3s
timeout: 2s
retries: 2
start_period: 20s
x-api-limits: &api-limits
limits:
cpus: "0.50"
# lab4: 4 одновременные загрузки по 10 MiB; замер пика — evidence/upload_memory*.json
memory: 320M
x-api-secrets: &api-secrets
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
services:
postgres:
image: postgres:16.4-alpine
restart: unless-stopped
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
# (API_REPLICAS + 1 при rolling update) x DB_POOL_MAX + migrate + запас < 100
command: ["postgres", "-c", "max_connections=100"]
secrets:
- postgres_password
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
interval: 3s
timeout: 3s
retries: 20
start_period: 10s
networks: [backend]
stop_grace_period: 30s
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
redis:
image: redis:7.4.1-alpine
restart: unless-stopped
user: redis
# Только кэш проверок сессии: без RDB/AOF, ограничение памяти и LRU.
command:
- sh
- -c
- exec redis-server --requirepass "$$(cat /run/secrets/redis_password)" --save "" --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru
secrets:
- redis_password
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"]
interval: 3s
timeout: 3s
retries: 20
networks: [backend]
deploy:
resources:
limits:
cpus: "0.25"
memory: 96M
# lab4: S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 gateway в
# одном процессе `weed server`). Наружу НЕ публикуется. Административные HTTP-порты
# (master 9333, filer 8888, volume 8080) слушают только 127.0.0.1 ВНУТРИ контейнера
# (-ip.bind), в сеть storage выставлен только S3 API (8333). Анонимного доступа нет:
# identities из secrets/s3_config (bootstrap-admin и messenger-api с правами
# Read/Write/List только на свой бакет). Bucket создаёт s3-init.
s3:
image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION:-4.50}
restart: unless-stopped
entrypoint: ["/usr/bin/weed"]
command:
- server
- -dir=/data
- -ip=127.0.0.1
- -ip.bind=127.0.0.1
- -s3
- -s3.port=8333
- -s3.ip.bind=0.0.0.0
- -s3.config=/run/secrets/s3_config
- -s3.iam=false
- -s3.port.iceberg=0
- -s3.port.lance=0
- -s3.autoCreateBucket=false
- -master.volumeSizeLimitMB=256
- -master.electionTimeout=2s
- -master.telemetry=false
- -volume.max=0
secrets:
- s3_config
volumes:
- s3data:/data
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"]
interval: 3s
timeout: 3s
retries: 20
start_period: 10s
networks: [storage]
# Штатная остановка SeaweedFS длится ~20 с; Docker ждёт 10 с и шлёт SIGKILL.
# Подтверждённые записи уже в page cache ОС, том консистентен (см. REPORT.md).
stop_grace_period: 10s
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
# Идемпотентный bootstrap бакета + проверка приватности и прав API-ключей.
# Повторный запуск: "already exists (idempotent no-op)". Admin-ключи есть только здесь.
s3-init:
build: .
command: ["python", "-m", "src.s3_bootstrap"]
restart: "no"
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_access_key_id
- s3_secret_access_key
depends_on:
s3:
condition: service_healthy
networks: [storage]
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.50"
memory: 192M
# Однократные миграции до старта реплик. Повтор — no-op (schema_migrations),
# параллельный запуск сериализуется pg_advisory_lock.
migrate:
build: .
command: ["python", "-m", "src.migrate"]
restart: "no"
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3}
# DDL может идти дольше обычного запроса API
DB_STATEMENT_TIMEOUT_MS: "60000"
MIGRATE_WAIT_SECONDS: "60"
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
networks: [backend]
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
api:
build: .
image: messenger-api:${APP_VERSION:-lab4}
restart: unless-stopped
environment: *api-env
secrets: *api-secrets
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
# Порт только внутри сети backend — в обход LB API недоступен.
expose:
- "8080"
healthcheck: *api-healthcheck
# SIGTERM -> осушение DRAIN_SECONDS (3 с) -> graceful shutdown (<= 10 с).
stop_signal: SIGTERM
stop_grace_period: 20s
read_only: true
tmpfs:
- /tmp:size=16m
networks: [backend, storage]
# Остановленный s3 исчезает из Docker DNS; короткий таймаут резолвера.
dns_opt: ["timeout:1", "attempts:1"]
deploy:
replicas: ${API_REPLICAS:-2}
resources: *api-limits
# Экземпляр с коротким TTL — только для автотеста истечения сессии (профиль ttl).
# Не входит в LB (другое DNS-имя), портов наружу не публикует: тест запускается
# внутри сети командой `make test-expiry`
api-short-ttl:
image: messenger-api:${APP_VERSION:-lab4}
profiles: [ttl]
restart: "no"
environment:
<<: *api-env
SESSION_TTL_SECONDS: ${SHORT_SESSION_TTL_SECONDS:-2}
secrets: *api-secrets
depends_on:
api:
condition: service_healthy
expose:
- "8080"
healthcheck: *api-healthcheck
stop_grace_period: 20s
read_only: true
tmpfs:
- /tmp:size=16m
networks: [backend, storage]
deploy:
resources: *api-limits
# lab4: экземпляры с ВНЕДРЁННЫМИ отказами пути загрузки (профиль faults), только для
# student-tests/test_attachment_consistency.py. Не входят в LB (другие DNS-имена),
# порты не публикуют. Запуск: python scripts/ops.py in-network consistency
api-fault-s3:
image: messenger-api:${APP_VERSION:-lab4}
profiles: [faults]
restart: "no"
environment:
<<: *api-env
FAULT_INJECTION: s3_error_on_put
secrets: *api-secrets
depends_on:
api:
condition: service_healthy
expose: ["8080"]
read_only: true
tmpfs: ["/tmp:size=16m"]
networks: [backend, storage]
deploy:
resources: *api-limits
api-fault-db:
image: messenger-api:${APP_VERSION:-lab4}
profiles: [faults]
restart: "no"
environment:
<<: *api-env
FAULT_INJECTION: db_error_after_s3_put
secrets: *api-secrets
depends_on:
api:
condition: service_healthy
expose: ["8080"]
read_only: true
tmpfs: ["/tmp:size=16m"]
networks: [backend, storage]
deploy:
resources: *api-limits
# Процесс завершается (os._exit) между S3 PUT и COMMIT — «падение» реплики.
api-fault-crash:
image: messenger-api:${APP_VERSION:-lab4}
profiles: [faults]
restart: "no"
environment:
<<: *api-env
FAULT_INJECTION: crash_after_s3_put
secrets: *api-secrets
depends_on:
api:
condition: service_healthy
expose: ["8080"]
read_only: true
tmpfs: ["/tmp:size=16m"]
networks: [backend, storage]
deploy:
resources: *api-limits
lb:
image: haproxy:2.8.30-alpine
restart: unless-stopped
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8404/lb-health"]
interval: 5s
timeout: 2s
retries: 3
stop_grace_period: 5s
networks: [backend]
deploy:
resources:
limits:
cpus: "0.50"
memory: 128M
volumes:
pgdata:
s3data:
networks:
backend:
# Внутренняя сеть хранилища объектов: без выхода наружу и без LB.
storage:
internal: true
secrets:
postgres_password:
file: ./secrets/postgres_password
redis_password:
file: ./secrets/redis_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_config:
file: ./secrets/s3_config.json
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key