Files
mtusi2026lab5/compose.yaml
T
celis a4a7b9d41a Реализовать лабораторную 5: очередь RabbitMQ, worker и outbox
- upload возвращает 202 после записи оригинала в staging/ и одной
  транзакции attachments + jobs + outbox
- relay outbox → RabbitMQ (quorum-очередь, publisher confirms) в каждой
  реплике API; sweeper возвращает работу с истёкшей арендой
- worker: аренда с fencing-токеном, варианты thumbnail/crop/optimized
  (sharp), retry с backoff и jitter, реестр failed, replay с аудитом,
  reprocess новой версией pipeline
- reconciler дочищает staging/ и variants/
- сценарии уровней 3–4, fault-injection трека 2 (14 сценариев,
  инварианты I1–I6), evidence и REPORT.md
2026-10-31 00:31:54 +03:00

258 lines
11 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Клиент → lb (HAProxy) → api × N → postgres (метаданные, jobs, outbox) + s3 (SeaweedFS, staging/ оригинала).
# api (relay outbox) → rabbitmq (quorum-очередь attachments.process) → worker × M → s3 (originals/, variants/) + postgres.
# Запуск: scripts/init-secrets.sh && docker compose --env-file .env up -d --build (см. REPORT.md)
name: messenger-lab5
x-app: &app
build: .
image: messenger-lab5-app:dev # уникальное имя: на общем Docker-host не перепутать с чужими стендами
networks: [backend]
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
x-jobs-env: &jobs-env
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-2000}
JOB_REPUBLISH_SECONDS: ${JOB_REPUBLISH_SECONDS:-600}
FAILPOINTS: ${FAILPOINTS:-0} # 1 только на тестовом стенде: файл failpoint-<точка>.png один раз роняет процесс
x-seaweedfs: &seaweedfs
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
restart: unless-stopped
stop_grace_period: 20s
services:
postgres:
image: postgres:18.6-alpine
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=256MB"]
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 15
deploy:
resources:
limits: { cpus: "${DB_CPUS:-2}", memory: 1g }
stop_grace_period: 30s
restart: unless-stopped
networks: [backend]
# Хранилище SeaweedFS: master + volume + filer, данные в volume s3data. Только во внутренней сети storage:
# его HTTP-интерфейсы (master :9333, filer :8888, volume :8080) не требуют подписи S3 и недоступны даже из API.
seaweedfs:
<<: *seaweedfs
command: ["server", "-dir=/data", "-ip=seaweedfs", "-ip.bind=0.0.0.0", "-filer", "-master.volumeSizeLimitMB=1024"]
volumes: [s3data:/data]
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:8888/"]
interval: 2s
timeout: 2s
retries: 30
start_period: 5s
deploy:
resources:
limits: { cpus: "1", memory: 512m }
networks: [storage]
# S3-шлюз — единственный путь к байтам: проверяет SigV4 и права identities (secrets/s3_identities.json).
# Встроенный IAM API, Iceberg и Lance выключены; порт не опубликован на хост.
s3:
<<: *seaweedfs
# concurrentUploadLimitMB: шлюз буферизует чанки (4 MiB) каждой загрузки; сверх лимита новые загрузки ждут,
# а не выводят шлюз за лимит памяти (эксперимент research/upload_memory.py: при 256m без лимита — OOM).
command: ["s3", "-filer=seaweedfs:8888", "-port=9000", "-config=/run/secrets/s3_identities",
"-iam=false", "-port.iceberg=0", "-port.lance=0", "-concurrentUploadLimitMB=256"]
secrets: [s3_identities]
depends_on:
seaweedfs: { condition: service_healthy }
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:9000/healthz"]
interval: 2s
timeout: 2s
retries: 30
deploy:
resources:
limits: { cpus: "1", memory: 768m }
networks: [storage, backend]
# Одноразовые сервисы до старта реплик: схема БД (advisory lock) и bucket (повторный запуск ничего не меняет).
migrate:
<<: *app
command: ["node", "src/main.ts", "migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
depends_on:
postgres: { condition: service_healthy }
s3-init:
<<: *app
command: ["node", "src/main.ts", "s3-init"]
environment: *s3-env
secrets:
- { source: s3_admin_access_key_id, target: s3_access_key_id }
- { source: s3_admin_secret_access_key, target: s3_secret_access_key }
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
depends_on:
s3: { condition: service_healthy }
# Брокер: RabbitMQ, quorum-очередь на диске (volume rabbitmq-data), publisher confirms, ручной ack.
# hostname фиксирует имя узла rabbit@rabbitmq: данные в volume привязаны к нему и переживают пересоздание.
rabbitmq:
image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5
hostname: rabbitmq
volumes:
- rabbitmq-data:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
secrets: [rabbitmq_definitions]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 30
start_period: 20s
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
networks: [backend]
api:
<<: *app
environment:
<<: [*s3-env, *jobs-env]
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-2500}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
secrets:
- database_url
- { source: broker_url_api, target: broker_url }
- { source: s3_api_access_key_id, target: s3_access_key_id }
- { source: s3_api_secret_access_key, target: s3_secret_access_key }
# rabbitmq не в depends_on: API принимает загрузки и без брокера, задания ждут в outbox.
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-256m}" }
stop_grace_period: 15s
restart: unless-stopped # упавшая реплика поднимается сама; после `docker stop` остаётся остановленной
# Worker: отдельный процесс без HTTP, конкурирующие consumers одной очереди. WORKER_CONCURRENCY = prefetch =
# одновременных заданий на реплику; память ограничена контейнером, libvips — один поток, без кэша.
worker:
<<: *app
command: ["node", "src/main.ts", "worker"]
environment:
<<: [*s3-env, *jobs-env]
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
secrets:
- database_url
- { source: broker_url_worker, target: broker_url }
- { source: s3_worker_access_key_id, target: s3_access_key_id }
- { source: s3_worker_secret_access_key, target: s3_secret_access_key }
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
rabbitmq: { condition: service_healthy }
deploy:
replicas: ${WORKER_REPLICAS:-2}
resources:
limits: { cpus: "${WORKER_CPUS:-1}", memory: "${WORKER_MEMORY:-768m}" }
stop_grace_period: 15s # SIGTERM: consumer отменяется, текущие задания до 10 с, затем выход
restart: unless-stopped
# Периодическая сверка БД ↔ S3 с grace period (src/reconcile.ts). Свои S3-ключи: объекты + листинг bucket.
reconciler:
<<: *app
command: ["node", "src/main.ts", "reconcile", "--apply", "--every=${RECONCILE_INTERVAL_SECONDS:-300}"]
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets:
- database_url
- { source: s3_reconciler_access_key_id, target: s3_access_key_id }
- { source: s3_reconciler_secret_access_key, target: s3_secret_access_key }
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
restart: unless-stopped
lb:
image: haproxy:3.2.26-alpine
volumes: ["./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # статистика HAProxy для оператора, это не API
depends_on:
api: { condition: service_healthy }
deploy:
resources:
limits: { cpus: "${LB_CPUS:-1}", memory: 128m }
restart: unless-stopped
networks: [frontend, backend]
networks:
frontend: {}
backend: { internal: true } # api, worker, postgres, rabbitmq и s3-шлюз недоступны с хоста и не имеют выхода наружу
storage: { internal: true } # внутренние компоненты SeaweedFS видит только s3-шлюз
volumes:
pgdata: {}
s3data: {}
rabbitmq-data: {}
secrets:
database_url: { file: ./secrets/database_url }
postgres_password: { file: ./secrets/postgres_password }
s3_identities: { file: ./secrets/s3_identities.json }
s3_admin_access_key_id: { file: ./secrets/s3_admin_access_key_id }
s3_admin_secret_access_key: { file: ./secrets/s3_admin_secret_access_key }
s3_api_access_key_id: { file: ./secrets/s3_api_access_key_id }
s3_api_secret_access_key: { file: ./secrets/s3_api_secret_access_key }
s3_reconciler_access_key_id: { file: ./secrets/s3_reconciler_access_key_id }
s3_reconciler_secret_access_key: { file: ./secrets/s3_reconciler_secret_access_key }
s3_worker_access_key_id: { file: ./secrets/s3_worker_access_key_id }
s3_worker_secret_access_key: { file: ./secrets/s3_worker_secret_access_key }
rabbitmq_definitions: { file: ./secrets/rabbitmq_definitions.json }
broker_url_api: { file: ./secrets/broker_url_api }
broker_url_worker: { file: ./secrets/broker_url_worker }