forked from mtusi2026/mtusi2026lab5
RabbitMQ (quorum queue, publisher confirms, manual ack, DLX) + отдельный worker: staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued; relay outbox, lease с heartbeat и fencing (worker_id, attempt), retry с backoff+jitter, failed-реестр, безопасный replay с audit; thumbnail/crop/WebP в изолированном процессе с лимитами. Тесты окон отказа (fault injection), трек 1 «Очередь под перегрузкой» с backpressure. Перенесено исправление замечания lab3 PR #99 (U+0000 в тексте). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
335 lines
12 KiB
YAML
335 lines
12 KiB
YAML
# Стенд lab5: lb (HAProxy) -> api x N -> postgres (метаданные, jobs, outbox) + s3 (RustFS: staging/, originals/,
|
||
# variants/) + rabbitmq (очередь заданий) <- worker (обработка изображений) ; valkey — кэш сессий.
|
||
# migrate и s3-init — одноразовый bootstrap схемы БД и приватного bucket.
|
||
# Подготовка: sh scripts/init-secrets.sh
|
||
# Запуск: docker compose --env-file .env up -d --build --wait
|
||
name: mtusi-lab5
|
||
|
||
x-api-env: &api-env
|
||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||
APP_PORT: "8080"
|
||
APP_ENV: ${APP_ENV:-lab}
|
||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||
SESSION_STORE: ${SESSION_STORE:-postgres+valkey}
|
||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
|
||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
|
||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
|
||
DB_ACQUIRE_TIMEOUT_MS: ${DB_ACQUIRE_TIMEOUT_MS:-2000}
|
||
CACHE_TIMEOUT_MS: ${CACHE_TIMEOUT_MS:-300}
|
||
PASSWORD_HASH_WORKERS: ${PASSWORD_HASH_WORKERS:-2}
|
||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||
S3_REGION: ${S3_REGION:-us-east-1}
|
||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
|
||
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
|
||
S3_TIMEOUT_MS: ${S3_TIMEOUT_MS:-5000}
|
||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||
IMAGE_CHECK_WORKERS: ${IMAGE_CHECK_WORKERS:-2}
|
||
BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url}
|
||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||
RETRY_BASE_MS: ${RETRY_BASE_MS:-1000}
|
||
RETRY_MAX_MS: ${RETRY_MAX_MS:-60000}
|
||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
|
||
OUTBOX_POLL_MS: ${OUTBOX_POLL_MS:-500}
|
||
WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640}
|
||
|
||
x-api-depends: &api-depends
|
||
postgres:
|
||
condition: service_healthy
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
# bucket создан и проверен на приватность до старта API
|
||
condition: service_completed_successfully
|
||
rabbitmq:
|
||
# Брокер не критичен для приёма: API пишет намерение в outbox и допубликует позже.
|
||
condition: service_started
|
||
required: false
|
||
valkey:
|
||
# Valkey — кэш, не источник истины: API должен стартовать и без него.
|
||
condition: service_started
|
||
required: false
|
||
|
||
x-api-limits: &api-limits
|
||
limits:
|
||
cpus: "${API_CPUS:-1.0}"
|
||
memory: ${API_MEMORY:-384M} # 2 Argon2id x 64 MiB + 2 загрузки x (10 MiB тело + проверка) + интерпретатор
|
||
|
||
services:
|
||
lb:
|
||
image: haproxy:3.2.8-alpine
|
||
volumes:
|
||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||
ports:
|
||
# Единственная публичная точка входа API (только loopback хоста) + страница состояния LB.
|
||
- "127.0.0.1:${LB_PUBLISHED_PORT:-8080}:8080"
|
||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||
networks: [edge]
|
||
healthcheck:
|
||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"]
|
||
interval: 5s
|
||
timeout: 2s
|
||
retries: 5
|
||
depends_on:
|
||
api:
|
||
condition: service_healthy
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.0", memory: 128M}
|
||
restart: unless-stopped
|
||
|
||
api:
|
||
image: mtusi-lab5-api:local
|
||
build: .
|
||
# Нет container_name и ports: реплики взаимозаменяемы, доступны только через lb.
|
||
deploy:
|
||
replicas: ${API_REPLICAS:-2}
|
||
resources: *api-limits
|
||
environment: *api-env
|
||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||
networks: [edge, data]
|
||
# Корневая ФС только для чтения, tmpfs /tmp — у API нет места для локальной копии оригинала.
|
||
read_only: true
|
||
tmpfs: [/tmp]
|
||
stop_grace_period: 20s # drain 3 с + graceful shutdown uvicorn до 10 с, с запасом
|
||
depends_on: *api-depends
|
||
restart: unless-stopped
|
||
|
||
migrate:
|
||
# Один запуск на весь стенд (а не в каждой реплике): pg_advisory_lock + schema_migrations.
|
||
build: .
|
||
image: mtusi-lab5-api:local
|
||
command: ["python", "-m", "app.migrate"]
|
||
environment:
|
||
DATABASE_URL_FILE: /run/secrets/database_url
|
||
SESSION_STORE: postgres
|
||
secrets: [database_url]
|
||
networks: [data]
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.5", memory: 128M}
|
||
restart: "no"
|
||
|
||
# S3-совместимое хранилище оригиналов. Порты не публикуются на хост: доступ только из сети data
|
||
# (API и s3-init). Веб-консоль RustFS выключена (по умолчанию, RUSTFS_CONSOLE_ENABLE не задан).
|
||
s3:
|
||
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
|
||
environment:
|
||
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id
|
||
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key
|
||
RUSTFS_REGION: ${S3_REGION:-us-east-1}
|
||
RUSTFS_VOLUMES: /data
|
||
RUSTFS_OBS_LOG_DIRECTORY: "" # логи в stdout контейнера
|
||
RUSTFS_OBS_LOGGER_LEVEL: warn
|
||
secrets: [s3_access_key_id, s3_secret_access_key]
|
||
volumes:
|
||
- s3data:/data
|
||
networks: [data]
|
||
healthcheck:
|
||
test: ["CMD", "curl", "-fs", "-o", "/dev/null", "http://127.0.0.1:9000/health"]
|
||
interval: 2s
|
||
timeout: 3s
|
||
retries: 30
|
||
start_period: 5s
|
||
stop_grace_period: 20s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.0", memory: 512M}
|
||
restart: unless-stopped
|
||
|
||
s3-init:
|
||
# Идемпотентный bootstrap: bucket создаётся, если его нет; анонимная политика удаляется;
|
||
# анонимный GET проверяется на отказ. Повторный запуск данные не трогает.
|
||
image: mtusi-lab5-api:local
|
||
command: ["python", "-m", "app.s3_bootstrap"]
|
||
environment:
|
||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||
S3_REGION: ${S3_REGION:-us-east-1}
|
||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||
secrets: [s3_access_key_id, s3_secret_access_key]
|
||
networks: [data]
|
||
depends_on:
|
||
s3:
|
||
condition: service_healthy
|
||
migrate:
|
||
condition: service_completed_successfully # образ api уже собран сервисом migrate
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.5", memory: 128M}
|
||
restart: "no"
|
||
|
||
# Брокер заданий. Quorum queue + persistent messages на volume rabbitdata переживают рестарт.
|
||
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
|
||
# Порты не публикуются; management UI не включён (только встроенный prometheus на 15692 в сети data).
|
||
rabbitmq:
|
||
image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5
|
||
hostname: rabbitmq
|
||
secrets:
|
||
- source: rabbitmq_conf
|
||
target: /etc/rabbitmq/conf.d/90-credentials.conf
|
||
volumes:
|
||
- rabbitdata:/var/lib/rabbitmq
|
||
networks: [data]
|
||
healthcheck:
|
||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||
interval: 5s
|
||
timeout: 10s
|
||
retries: 30
|
||
start_period: 10s
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.0", memory: 512M}
|
||
restart: unless-stopped
|
||
|
||
# Worker: отдельный процесс обработки (тот же образ, другая команда). Не в сети edge, портов нет.
|
||
worker:
|
||
image: mtusi-lab5-api:local
|
||
command: ["python", "-m", "app.worker"]
|
||
environment:
|
||
<<: *api-env
|
||
SESSION_STORE: postgres # worker не работает с сессиями
|
||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||
networks: [data]
|
||
read_only: true
|
||
tmpfs: [/tmp]
|
||
healthcheck:
|
||
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-alive')<20 else 1)"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 20s
|
||
stop_grace_period: 70s # дождаться текущих заданий (JOB_TIMEOUT_SECONDS)
|
||
deploy:
|
||
replicas: ${WORKER_REPLICAS:-1}
|
||
resources:
|
||
limits:
|
||
cpus: "${WORKER_CPUS:-2.0}"
|
||
memory: ${WORKER_MEMORY:-1536M} # WORKER_CONCURRENCY x WORKER_TASK_MEMORY_MB + процесс worker
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
condition: service_completed_successfully
|
||
rabbitmq:
|
||
condition: service_healthy
|
||
restart: unless-stopped
|
||
|
||
postgres:
|
||
image: postgres:17.7-bookworm
|
||
environment:
|
||
POSTGRES_USER: app
|
||
POSTGRES_DB: messenger
|
||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||
secrets: [postgres_password]
|
||
volumes:
|
||
- pgdata:/var/lib/postgresql/data
|
||
networks: [data]
|
||
healthcheck:
|
||
test: ["CMD", "pg_isready", "-U", "app", "-d", "messenger"]
|
||
interval: 2s
|
||
timeout: 3s
|
||
retries: 30
|
||
shm_size: 128m
|
||
stop_grace_period: 30s # даём PostgreSQL завершиться с checkpoint, без recovery при старте
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "${PG_CPUS:-2.0}", memory: 768M}
|
||
restart: unless-stopped
|
||
|
||
valkey:
|
||
image: valkey/valkey:8.1.4
|
||
command: ["valkey-server", "/run/secrets/valkey_conf"]
|
||
secrets: [valkey_conf]
|
||
networks: [data]
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_conf)\" ping | grep -q PONG"]
|
||
interval: 2s
|
||
timeout: 3s
|
||
retries: 30
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.5", memory: 96M}
|
||
restart: unless-stopped
|
||
|
||
# Отдельная реплика с коротким TTL только для автотеста истечения сессии (lab2, уровень 4).
|
||
# Порт не публикуется: tester обращается к ней по внутренней сети.
|
||
api-ttl:
|
||
profiles: [test]
|
||
image: mtusi-lab5-api:local
|
||
environment:
|
||
<<: *api-env
|
||
SESSION_TTL_SECONDS: "2"
|
||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||
networks: [edge, data]
|
||
read_only: true
|
||
tmpfs: [/tmp]
|
||
deploy:
|
||
resources: *api-limits
|
||
depends_on: *api-depends
|
||
|
||
# Linux-окружение для make/тестов/hook-скриптов (нужно на Windows без make и WSL).
|
||
tester:
|
||
profiles: [tools]
|
||
build:
|
||
context: .
|
||
dockerfile: tools/tester.Dockerfile
|
||
image: mtusi-lab5-tester:local
|
||
working_dir: /work
|
||
environment:
|
||
BASE_URL: http://lb:8080
|
||
TTL_BASE_URL: http://api-ttl:8080
|
||
LB_STATS_URL: http://lb:8404/stats
|
||
PYTHONUTF8: "1"
|
||
PYTHONDONTWRITEBYTECODE: "1"
|
||
networks: [edge]
|
||
volumes:
|
||
- .:/work
|
||
- /var/run/docker.sock:/var/run/docker.sock
|
||
|
||
networks:
|
||
# edge: lb <-> api (+ tester). data: api/migrate <-> хранилища; lb в неё не входит.
|
||
edge:
|
||
data:
|
||
internal: true
|
||
|
||
volumes:
|
||
pgdata:
|
||
s3data:
|
||
rabbitdata:
|
||
|
||
secrets:
|
||
postgres_password:
|
||
file: ./secrets/postgres_password
|
||
database_url:
|
||
file: ./secrets/database_url
|
||
cache_url:
|
||
file: ./secrets/cache_url
|
||
valkey_conf:
|
||
file: ./secrets/valkey.conf
|
||
s3_access_key_id:
|
||
file: ./secrets/s3_access_key_id
|
||
s3_secret_access_key:
|
||
file: ./secrets/s3_secret_access_key
|
||
broker_url:
|
||
file: ./secrets/broker_url
|
||
rabbitmq_conf:
|
||
file: ./secrets/rabbitmq.conf
|