Files
mtusi2026lab5/compose.yaml
T
RainakandClaude Opus 5.5 0a221b0675 ЛР5 БВТ2304 Запорожец на 5
RabbitMQ (quorum queue, publisher confirms, manual ack, DLX) + отдельный worker:
staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued; relay outbox,
lease с heartbeat и fencing (worker_id, attempt), retry с backoff+jitter, failed-реестр,
безопасный replay с audit; thumbnail/crop/WebP в изолированном процессе с лимитами.
Тесты окон отказа (fault injection), трек 1 «Очередь под перегрузкой» с backpressure.
Перенесено исправление замечания lab3 PR #99 (U+0000 в тексте).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 18:33:52 +03:00

335 lines
12 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Стенд lab5: lb (HAProxy) -> api x N -> postgres (метаданные, jobs, outbox) + s3 (RustFS: staging/, originals/,
# variants/) + rabbitmq (очередь заданий) <- worker (обработка изображений) ; valkey — кэш сессий.
# migrate и s3-init — одноразовый bootstrap схемы БД и приватного bucket.
# Подготовка: sh scripts/init-secrets.sh
# Запуск: docker compose --env-file .env up -d --build --wait
name: mtusi-lab5
x-api-env: &api-env
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-postgres+valkey}
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
DB_ACQUIRE_TIMEOUT_MS: ${DB_ACQUIRE_TIMEOUT_MS:-2000}
CACHE_TIMEOUT_MS: ${CACHE_TIMEOUT_MS:-300}
PASSWORD_HASH_WORKERS: ${PASSWORD_HASH_WORKERS:-2}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
S3_TIMEOUT_MS: ${S3_TIMEOUT_MS:-5000}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
IMAGE_CHECK_WORKERS: ${IMAGE_CHECK_WORKERS:-2}
BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
RETRY_BASE_MS: ${RETRY_BASE_MS:-1000}
RETRY_MAX_MS: ${RETRY_MAX_MS:-60000}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
OUTBOX_POLL_MS: ${OUTBOX_POLL_MS:-500}
WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640}
x-api-depends: &api-depends
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
# bucket создан и проверен на приватность до старта API
condition: service_completed_successfully
rabbitmq:
# Брокер не критичен для приёма: API пишет намерение в outbox и допубликует позже.
condition: service_started
required: false
valkey:
# Valkey — кэш, не источник истины: API должен стартовать и без него.
condition: service_started
required: false
x-api-limits: &api-limits
limits:
cpus: "${API_CPUS:-1.0}"
memory: ${API_MEMORY:-384M} # 2 Argon2id x 64 MiB + 2 загрузки x (10 MiB тело + проверка) + интерпретатор
services:
lb:
image: haproxy:3.2.8-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
# Единственная публичная точка входа API (только loopback хоста) + страница состояния LB.
- "127.0.0.1:${LB_PUBLISHED_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
networks: [edge]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"]
interval: 5s
timeout: 2s
retries: 5
depends_on:
api:
condition: service_healthy
deploy:
resources:
limits: {cpus: "1.0", memory: 128M}
restart: unless-stopped
api:
image: mtusi-lab5-api:local
build: .
# Нет container_name и ports: реплики взаимозаменяемы, доступны только через lb.
deploy:
replicas: ${API_REPLICAS:-2}
resources: *api-limits
environment: *api-env
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
networks: [edge, data]
# Корневая ФС только для чтения, tmpfs /tmp — у API нет места для локальной копии оригинала.
read_only: true
tmpfs: [/tmp]
stop_grace_period: 20s # drain 3 с + graceful shutdown uvicorn до 10 с, с запасом
depends_on: *api-depends
restart: unless-stopped
migrate:
# Один запуск на весь стенд (а не в каждой реплике): pg_advisory_lock + schema_migrations.
build: .
image: mtusi-lab5-api:local
command: ["python", "-m", "app.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_STORE: postgres
secrets: [database_url]
networks: [data]
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: "no"
# S3-совместимое хранилище оригиналов. Порты не публикуются на хост: доступ только из сети data
# (API и s3-init). Веб-консоль RustFS выключена (по умолчанию, RUSTFS_CONSOLE_ENABLE не задан).
s3:
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
environment:
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key
RUSTFS_REGION: ${S3_REGION:-us-east-1}
RUSTFS_VOLUMES: /data
RUSTFS_OBS_LOG_DIRECTORY: "" # логи в stdout контейнера
RUSTFS_OBS_LOGGER_LEVEL: warn
secrets: [s3_access_key_id, s3_secret_access_key]
volumes:
- s3data:/data
networks: [data]
healthcheck:
test: ["CMD", "curl", "-fs", "-o", "/dev/null", "http://127.0.0.1:9000/health"]
interval: 2s
timeout: 3s
retries: 30
start_period: 5s
stop_grace_period: 20s
deploy:
resources:
limits: {cpus: "1.0", memory: 512M}
restart: unless-stopped
s3-init:
# Идемпотентный bootstrap: bucket создаётся, если его нет; анонимная политика удаляется;
# анонимный GET проверяется на отказ. Повторный запуск данные не трогает.
image: mtusi-lab5-api:local
command: ["python", "-m", "app.s3_bootstrap"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully # образ api уже собран сервисом migrate
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: "no"
# Брокер заданий. Quorum queue + persistent messages на volume rabbitdata переживают рестарт.
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
# Порты не публикуются; management UI не включён (только встроенный prometheus на 15692 в сети data).
rabbitmq:
image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5
hostname: rabbitmq
secrets:
- source: rabbitmq_conf
target: /etc/rabbitmq/conf.d/90-credentials.conf
volumes:
- rabbitdata:/var/lib/rabbitmq
networks: [data]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 30
start_period: 10s
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.0", memory: 512M}
restart: unless-stopped
# Worker: отдельный процесс обработки (тот же образ, другая команда). Не в сети edge, портов нет.
worker:
image: mtusi-lab5-api:local
command: ["python", "-m", "app.worker"]
environment:
<<: *api-env
SESSION_STORE: postgres # worker не работает с сессиями
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
networks: [data]
read_only: true
tmpfs: [/tmp]
healthcheck:
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-alive')<20 else 1)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 20s
stop_grace_period: 70s # дождаться текущих заданий (JOB_TIMEOUT_SECONDS)
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits:
cpus: "${WORKER_CPUS:-2.0}"
memory: ${WORKER_MEMORY:-1536M} # WORKER_CONCURRENCY x WORKER_TASK_MEMORY_MB + процесс worker
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
restart: unless-stopped
postgres:
image: postgres:17.7-bookworm
environment:
POSTGRES_USER: app
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql/data
networks: [data]
healthcheck:
test: ["CMD", "pg_isready", "-U", "app", "-d", "messenger"]
interval: 2s
timeout: 3s
retries: 30
shm_size: 128m
stop_grace_period: 30s # даём PostgreSQL завершиться с checkpoint, без recovery при старте
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: 768M}
restart: unless-stopped
valkey:
image: valkey/valkey:8.1.4
command: ["valkey-server", "/run/secrets/valkey_conf"]
secrets: [valkey_conf]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_conf)\" ping | grep -q PONG"]
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits: {cpus: "0.5", memory: 96M}
restart: unless-stopped
# Отдельная реплика с коротким TTL только для автотеста истечения сессии (lab2, уровень 4).
# Порт не публикуется: tester обращается к ней по внутренней сети.
api-ttl:
profiles: [test]
image: mtusi-lab5-api:local
environment:
<<: *api-env
SESSION_TTL_SECONDS: "2"
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
networks: [edge, data]
read_only: true
tmpfs: [/tmp]
deploy:
resources: *api-limits
depends_on: *api-depends
# Linux-окружение для make/тестов/hook-скриптов (нужно на Windows без make и WSL).
tester:
profiles: [tools]
build:
context: .
dockerfile: tools/tester.Dockerfile
image: mtusi-lab5-tester:local
working_dir: /work
environment:
BASE_URL: http://lb:8080
TTL_BASE_URL: http://api-ttl:8080
LB_STATS_URL: http://lb:8404/stats
PYTHONUTF8: "1"
PYTHONDONTWRITEBYTECODE: "1"
networks: [edge]
volumes:
- .:/work
- /var/run/docker.sock:/var/run/docker.sock
networks:
# edge: lb <-> api (+ tester). data: api/migrate <-> хранилища; lb в неё не входит.
edge:
data:
internal: true
volumes:
pgdata:
s3data:
rabbitdata:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
valkey_conf:
file: ./secrets/valkey.conf
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
broker_url:
file: ./secrets/broker_url
rabbitmq_conf:
file: ./secrets/rabbitmq.conf