Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
411 lines
15 KiB
YAML
411 lines
15 KiB
YAML
# Lab5 stand: API replicas behind HAProxy, PostgreSQL, Redis, SeaweedFS S3,
|
|
# a durable RabbitMQ broker, one outbox relay and a separate image worker.
|
|
#
|
|
# cp -n .env.example .env
|
|
# docker compose --env-file .env up -d --build
|
|
# curl http://localhost:8080/health/ready
|
|
#
|
|
# Only the load balancer is published (127.0.0.1). Stores and API replicas are
|
|
# reachable from the host via `docker compose exec` / `docker compose logs`.
|
|
# No container_name anywhere: replica hostnames stay the short container IDs,
|
|
# which the application reports back as X-Instance-Id.
|
|
name: mtusi-messenger-lab5
|
|
|
|
# The image is built once by the api service. migrate and s3-init only consume
|
|
# it; duplicating `build` here makes Compose compile the same Go tree in parallel
|
|
# and OOMs Docker Desktop's default VM.
|
|
x-app: &app
|
|
image: mtusi-messenger:lab5
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:18-alpine@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873
|
|
environment:
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
|
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
|
volumes:
|
|
- postgres18-data:/var/lib/postgresql
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 5s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
# Saturation point of the stand (see REPORT.md, track 1): the database
|
|
# runs at its quota while API replicas idle. Raise this knob to give the
|
|
# shared dependency more of the host CPU.
|
|
cpus: "${POSTGRES_CPU_LIMIT:-0.50}"
|
|
memory: ${POSTGRES_MEM_LIMIT:-192M}
|
|
restart: unless-stopped
|
|
|
|
redis:
|
|
image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
|
|
command:
|
|
- redis-server
|
|
- --appendonly
|
|
- "yes"
|
|
- --appendfsync
|
|
- everysec
|
|
- --save
|
|
- "60"
|
|
- "1"
|
|
volumes:
|
|
- redis7-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 10
|
|
start_period: 3s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "0.25"
|
|
memory: 96M
|
|
restart: unless-stopped
|
|
|
|
# Private S3-compatible object store. Only the S3 API is published, on
|
|
# loopback, so the presigned-upload track and evidence can reach it from the
|
|
# host. master/volume/filer ports stay inside the compose network.
|
|
seaweedfs:
|
|
image: chrislusf/seaweedfs:3.80@sha256:1055999e08eed1789b0ae45d235126e4495e23d3fb9d6396293fd42539b1ae6a
|
|
command:
|
|
- server
|
|
- -s3
|
|
- -s3.config=/etc/seaweedfs/s3.json
|
|
- -dir=/data
|
|
- -master.volumeSizeLimitMB=64
|
|
volumes:
|
|
- ./infra/s3/s3.json:/etc/seaweedfs/s3.json:ro
|
|
- seaweedfs-data:/data
|
|
environment:
|
|
# Keep the Go runtime below the container limit under concurrent PUTs.
|
|
GOGC: "50"
|
|
GOMEMLIMIT: "512MiB"
|
|
ports:
|
|
- "127.0.0.1:${S3_PORT:-8333}:8333"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 20
|
|
start_period: 60s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "1.00"
|
|
memory: 640M
|
|
restart: unless-stopped
|
|
|
|
|
|
# Durable job transport. Quorum queues, publisher confirms and a DLQ live in
|
|
# the broker; PostgreSQL's outbox remains the source of truth for intent.
|
|
broker:
|
|
image: rabbitmq:4.3.7-management-alpine@sha256:1528a88eebf2bc4d0ab7e9d5d9bc542e9bdf5e1b105e65848fcf8de5c2186528
|
|
hostname: broker
|
|
environment:
|
|
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
|
|
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-messenger}
|
|
RABBITMQ_DEFAULT_VHOST: ${BROKER_VHOST:-messenger}
|
|
volumes:
|
|
- rabbitmq-data:/var/lib/rabbitmq
|
|
ports:
|
|
# Management UI only on loopback, for operator evidence. AMQP stays internal.
|
|
- "127.0.0.1:${BROKER_MANAGEMENT_PORT:-15672}:15672"
|
|
healthcheck:
|
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 12
|
|
start_period: 30s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${BROKER_CPU_LIMIT:-0.50}"
|
|
memory: ${BROKER_MEM_LIMIT:-384M}
|
|
stop_grace_period: 30s
|
|
restart: unless-stopped
|
|
|
|
# One-shot, idempotent bucket bootstrap and identity probe.
|
|
s3-init:
|
|
<<: *app
|
|
entrypoint: ["/usr/local/bin/s3init"]
|
|
environment:
|
|
S3_ENDPOINT: http://seaweedfs:8333
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:-lab4admin}
|
|
S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:-lab4admin-secret}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret}
|
|
depends_on:
|
|
seaweedfs:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
disable: true
|
|
restart: "no"
|
|
|
|
# One-shot schema bootstrap; runs once to completion before any API replica.
|
|
migrate:
|
|
<<: *app
|
|
entrypoint: ["/usr/local/bin/migrate"]
|
|
command: ["up"]
|
|
environment:
|
|
APP_CONFIG: /etc/messenger/dev/config.yaml
|
|
POSTGRES_HOST: postgres
|
|
POSTGRES_PORT: "5432"
|
|
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
|
|
POSTGRES_SSL_MODE: disable
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "0.50"
|
|
memory: 256M
|
|
# One-shot: healthcheck из образа (проверка /health/live) для него бессмыслен.
|
|
healthcheck:
|
|
disable: true
|
|
restart: "no"
|
|
|
|
api:
|
|
<<: *app
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
# Общий сетевой alias: LB резолвит одно имя в адреса всех реплик, поэтому
|
|
# масштабирование и замена контейнеров не зависят от их номеров/ID.
|
|
networks:
|
|
default:
|
|
aliases:
|
|
- api-replica
|
|
env_file:
|
|
- path: .env
|
|
required: false
|
|
# Explicit values win over localhost defaults coming from .env (see .env.example).
|
|
environment:
|
|
APP_CONFIG: /etc/messenger/dev/config.yaml
|
|
POSTGRES_HOST: postgres
|
|
POSTGRES_PORT: "5432"
|
|
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
|
|
POSTGRES_SSL_MODE: disable
|
|
REDIS_ENABLED: "true"
|
|
REDIS_HOST: redis
|
|
REDIS_PORT: "6379"
|
|
CACHE_ENABLED: "true"
|
|
S3_ENDPOINT: http://seaweedfs:8333
|
|
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:8333}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret}
|
|
S3_USE_PATH_STYLE: "true"
|
|
S3_PRESIGN_TTL: ${S3_PRESIGN_TTL:-5m}
|
|
S3_SWEEP_INTERVAL: ${S3_SWEEP_INTERVAL:-1m}
|
|
S3_SWEEP_GRACE: ${S3_SWEEP_GRACE:-10m}
|
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
|
PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1}
|
|
PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5}
|
|
PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s}
|
|
PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s}
|
|
PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s}
|
|
PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s}
|
|
PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s}
|
|
STAGING_ORPHAN_GRACE: ${STAGING_ORPHAN_GRACE:-1h}
|
|
STAGING_FAILED_RETENTION: ${STAGING_FAILED_RETENTION:-168h}
|
|
ADMISSION_MAX_PENDING_PER_USER: ${ADMISSION_MAX_PENDING_PER_USER:-20}
|
|
ADMISSION_MAX_PENDING: ${ADMISSION_MAX_PENDING:-200}
|
|
ADMISSION_MAX_STAGED_BYTES: ${ADMISSION_MAX_STAGED_BYTES:-2147483648}
|
|
ADMISSION_MAX_CONCURRENT_UPLOADS: ${ADMISSION_MAX_CONCURRENT_UPLOADS:-8}
|
|
# Размер пула соединений ОДНОЙ реплики; общий расход = реплики × значение.
|
|
# Это один из общих ресурсов, который масштабирование расходует первым.
|
|
POSTGRES_MAX_CONNS: "${POSTGRES_MAX_CONNS:-10}"
|
|
deploy:
|
|
replicas: ${API_REPLICAS:-2}
|
|
resources:
|
|
limits:
|
|
cpus: "${API_CPU_LIMIT:-0.50}"
|
|
memory: ${API_MEM_LIMIT:-256M}
|
|
# The app drains on SIGTERM within SERVER_SHUTDOWN_TIMEOUT; keep the grace
|
|
# period above it so Docker does not SIGKILL an in-flight request.
|
|
stop_grace_period: 15s
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
s3-init:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
|
interval: 10s
|
|
# Готовность проверяет PostgreSQL (критичную зависимость; кэш деградирует
|
|
# безопасно): под насыщающей нагрузкой ответ может подойти к секундам, и
|
|
# слишком строгий таймаут помечает занятую, но работающую реплику как
|
|
# unhealthy.
|
|
timeout: 5s
|
|
retries: 6
|
|
start_period: 5s
|
|
restart: unless-stopped
|
|
|
|
|
|
# Separate worker process. It is intentionally not published and can be
|
|
# stopped/scaled independently from the API; stopping it must not stop 202.
|
|
worker:
|
|
<<: *app
|
|
entrypoint: ["/usr/local/bin/worker"]
|
|
env_file:
|
|
- path: .env
|
|
required: false
|
|
environment:
|
|
APP_CONFIG: /etc/messenger/dev/config.yaml
|
|
POSTGRES_HOST: postgres
|
|
POSTGRES_PORT: "5432"
|
|
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
|
|
POSTGRES_SSL_MODE: disable
|
|
S3_ENDPOINT: http://seaweedfs:8333
|
|
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:8333}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-lab4app}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-lab4app-secret}
|
|
S3_USE_PATH_STYLE: "true"
|
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
|
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger}
|
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
|
|
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
|
WORKER_SHUTDOWN_TIMEOUT: ${WORKER_SHUTDOWN_TIMEOUT:-30s}
|
|
WORKER_HEALTH_ADDR: ":8081"
|
|
FAULTS_ENABLED: ${FAULTS_ENABLED:-0}
|
|
FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-}
|
|
FAULT_WORKER_AFTER_CLAIM: ${FAULT_WORKER_AFTER_CLAIM:-}
|
|
FAULT_WORKER_AFTER_FIRST_VARIANT: ${FAULT_WORKER_AFTER_FIRST_VARIANT:-}
|
|
FAULT_WORKER_AFTER_PUT: ${FAULT_WORKER_AFTER_PUT:-}
|
|
FAULT_WORKER_AFTER_COMMIT: ${FAULT_WORKER_AFTER_COMMIT:-}
|
|
FAULT_WORKER_FAIL: ${FAULT_WORKER_FAIL:-}
|
|
FAULT_WORKER_SLEEP_AFTER_CLAIM: ${FAULT_WORKER_SLEEP_AFTER_CLAIM:-}
|
|
PROCESSING_PIPELINE_VERSION: ${PROCESSING_PIPELINE_VERSION:-1}
|
|
PROCESSING_MAX_ATTEMPTS: ${PROCESSING_MAX_ATTEMPTS:-5}
|
|
PROCESSING_JOB_TIMEOUT: ${PROCESSING_JOB_TIMEOUT:-120s}
|
|
PROCESSING_LEASE_TTL: ${PROCESSING_LEASE_TTL:-180s}
|
|
PROCESSING_HEARTBEAT: ${PROCESSING_HEARTBEAT:-30s}
|
|
PROCESSING_RETRY_BASE: ${PROCESSING_RETRY_BASE:-2s}
|
|
PROCESSING_RETRY_CAP: ${PROCESSING_RETRY_CAP:-60s}
|
|
ports:
|
|
- "127.0.0.1:${WORKER_HEALTH_PORT:-8081}:8081"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
s3-init:
|
|
condition: service_completed_successfully
|
|
broker:
|
|
condition: service_healthy
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${WORKER_CPU_LIMIT:-1.00}"
|
|
memory: ${WORKER_MEM_LIMIT:-1G}
|
|
stop_grace_period: 40s
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8081/health/ready"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 12
|
|
start_period: 10s
|
|
restart: unless-stopped
|
|
|
|
# Outbox publisher + lease reaper. One replica: duplicate relays are safe but
|
|
# a single owner keeps the evidence and the claim reaper simple.
|
|
relay:
|
|
<<: *app
|
|
entrypoint: ["/usr/local/bin/relay"]
|
|
env_file:
|
|
- path: .env
|
|
required: false
|
|
environment:
|
|
APP_CONFIG: /etc/messenger/dev/config.yaml
|
|
POSTGRES_HOST: postgres
|
|
POSTGRES_PORT: "5432"
|
|
POSTGRES_DATABASE: ${POSTGRES_DB:-messenger}
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-messenger}
|
|
POSTGRES_SSL_MODE: disable
|
|
BROKER_URL: ${BROKER_URL:-amqp://messenger:messenger@broker:5672/messenger}
|
|
BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1}
|
|
BROKER_DLQ: ${BROKER_DLQ:-attachment.process.dead}
|
|
OUTBOX_POLL_INTERVAL: ${OUTBOX_POLL_INTERVAL:-1s}
|
|
OUTBOX_CLAIM_TTL: ${OUTBOX_CLAIM_TTL:-30s}
|
|
OUTBOX_BATCH: ${OUTBOX_BATCH:-20}
|
|
OUTBOX_REAPER_INTERVAL: ${OUTBOX_REAPER_INTERVAL:-15s}
|
|
OUTBOX_RECONCILE_AFTER: ${OUTBOX_RECONCILE_AFTER:-5m}
|
|
OUTBOX_RETENTION: ${OUTBOX_RETENTION:-168h}
|
|
RELAY_HEALTH_ADDR: ":8082"
|
|
FAULTS_ENABLED: ${FAULTS_ENABLED:-0}
|
|
FAULT_TARGET_JOB: ${FAULT_TARGET_JOB:-}
|
|
FAULT_OUTBOX_AFTER_COMMIT: ${FAULT_OUTBOX_AFTER_COMMIT:-}
|
|
ports:
|
|
- "127.0.0.1:${RELAY_HEALTH_PORT:-8082}:8082"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
broker:
|
|
condition: service_healthy
|
|
deploy:
|
|
replicas: 1
|
|
resources:
|
|
limits:
|
|
cpus: "${RELAY_CPU_LIMIT:-0.25}"
|
|
memory: ${RELAY_MEM_LIMIT:-128M}
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/health/ready"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 12
|
|
start_period: 10s
|
|
restart: unless-stopped
|
|
|
|
# Public entry point. Active /health/ready probes remove a not-ready replica from
|
|
# rotation entirely; infra/lb/haproxy.cfg holds the upstream, checks and retries.
|
|
lb:
|
|
image: haproxy:3.2-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
|
|
volumes:
|
|
- ./infra/lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
ports:
|
|
- "127.0.0.1:${SERVER_PORT:-8080}:80"
|
|
# Operator-only stats page on loopback: shows which replica is UP/DOWN and why.
|
|
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
|
depends_on:
|
|
api:
|
|
condition: service_healthy
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${LB_CPU_LIMIT:-0.25}"
|
|
memory: 48M
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
postgres18-data:
|
|
redis7-data:
|
|
seaweedfs-data:
|
|
rabbitmq-data:
|