forked from mtusi2026/mtusi2026lab5
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
337 lines
10 KiB
YAML
337 lines
10 KiB
YAML
name: mtusi2026lab
|
|
|
|
x-api-build: &api-build
|
|
image: mtusi2026lab-api:local
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
|
|
# API receives only the attachments/ scoped S3 credential
|
|
x-api-environment: &api-environment
|
|
APP_HOST: 0.0.0.0
|
|
APP_PORT: "8080"
|
|
DB_POOL_MAX_CONNS: ${DB_POOL_MAX_CONNS:-10}
|
|
DATABASE_URL: postgres://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}?sslmode=disable
|
|
CACHE_URL: redis://redis:6379/0
|
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
|
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://rustfs:9000}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
|
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
|
IMAGE_DECODE_MEMORY_MB: ${IMAGE_DECODE_MEMORY_MB:-256}
|
|
UPLOAD_TEMP_DIR: /tmp/messenger-uploads
|
|
# Lab5 async pipeline. The API publishes outbox rows; the worker consumes.
|
|
BROKER_URL: "amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}@rabbitmq:5672/"
|
|
JOB_QUEUE: ${JOB_QUEUE:-attachment.process.v1}
|
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
|
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-1000}
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:16.10-alpine
|
|
restart: unless-stopped
|
|
environment:
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
|
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
|
interval: 3s
|
|
timeout: 3s
|
|
retries: 20
|
|
start_period: 5s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${POSTGRES_CPU_LIMIT:-1.0}"
|
|
memory: "${POSTGRES_MEM_LIMIT:-512M}"
|
|
|
|
redis:
|
|
image: redis:7.4.7-alpine
|
|
restart: unless-stopped
|
|
command:
|
|
[
|
|
"redis-server",
|
|
"--appendonly",
|
|
"yes",
|
|
"--appendfsync",
|
|
"everysec",
|
|
"--save",
|
|
"",
|
|
]
|
|
volumes:
|
|
- redisdata:/data
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 3s
|
|
timeout: 3s
|
|
retries: 20
|
|
start_period: 3s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${REDIS_CPU_LIMIT:-0.50}"
|
|
memory: "${REDIS_MEM_LIMIT:-256M}"
|
|
|
|
rabbitmq:
|
|
image: rabbitmq:4.2-management-alpine
|
|
restart: unless-stopped
|
|
environment:
|
|
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
|
|
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}
|
|
volumes:
|
|
- rabbitmqdata:/var/lib/rabbitmq
|
|
ports:
|
|
# Management UI is only for the local learning stand.
|
|
- "127.0.0.1:${RABBITMQ_MGMT_PORT:-15672}:15672"
|
|
healthcheck:
|
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
|
interval: 3s
|
|
timeout: 5s
|
|
retries: 20
|
|
start_period: 10s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${RABBITMQ_CPU_LIMIT:-1.0}"
|
|
memory: "${RABBITMQ_MEM_LIMIT:-512M}"
|
|
|
|
rustfs:
|
|
image: rustfs/rustfs:1.0.4
|
|
restart: unless-stopped
|
|
# AGENT: no published ports. The S3 API is reachable only on the Compose
|
|
# network, and the admin console is disabled. For inspection, see the
|
|
# one-shot rc client pattern in scripts/check-private-bucket.sh.
|
|
environment:
|
|
RUSTFS_ACCESS_KEY: ${S3_ROOT_ACCESS_KEY:?set S3_ROOT_ACCESS_KEY in .env}
|
|
RUSTFS_SECRET_KEY: ${S3_ROOT_SECRET_KEY:?set S3_ROOT_SECRET_KEY in .env}
|
|
RUSTFS_ADDRESS: ":9000"
|
|
RUSTFS_CONSOLE_ENABLE: "false"
|
|
RUSTFS_OBS_LOGGER_LEVEL: "error"
|
|
RUSTFS_OBS_LOG_DIRECTORY: "/logs"
|
|
command: ["/data"]
|
|
volumes:
|
|
- rustfsdata:/data
|
|
expose:
|
|
- "9000"
|
|
healthcheck:
|
|
test:
|
|
[
|
|
"CMD-SHELL",
|
|
"wget -q -O - http://127.0.0.1:9000/health/ready >/dev/null",
|
|
]
|
|
interval: 3s
|
|
timeout: 3s
|
|
retries: 20
|
|
start_period: 5s
|
|
stop_grace_period: 15s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${RUSTFS_CPU_LIMIT:-1.0}"
|
|
memory: "${RUSTFS_MEM_LIMIT:-512M}"
|
|
|
|
# AGENT: bootstrap converges the bucket and the scoped IAM identity on every
|
|
# run. Create is idempotent (ignore-existing), policy create replaces the
|
|
# named document, and user add updates the password if the user exists.
|
|
s3-init:
|
|
image: rustfs/rc:v0.1.36
|
|
restart: "no"
|
|
depends_on:
|
|
rustfs:
|
|
condition: service_healthy
|
|
environment:
|
|
HOME: /tmp
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://rustfs:9000}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ROOT_ACCESS_KEY: ${S3_ROOT_ACCESS_KEY:?set S3_ROOT_ACCESS_KEY in .env}
|
|
S3_ROOT_SECRET_KEY: ${S3_ROOT_SECRET_KEY:?set S3_ROOT_SECRET_KEY in .env}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
|
|
S3_APP_USER: ${S3_APP_USER:-messenger-api}
|
|
S3_APP_POLICY: ${S3_APP_POLICY:-messenger-attachments-rw}
|
|
S3_POLICY_TEMPLATE: /config/api-policy.json.template
|
|
volumes:
|
|
- ./infra/rustfs/bootstrap.sh:/bootstrap.sh:ro
|
|
- ./infra/rustfs/api-policy.json.template:/config/api-policy.json.template:ro
|
|
entrypoint: ["/bin/sh", "/bootstrap.sh"]
|
|
|
|
migrate:
|
|
<<: *api-build
|
|
restart: "no"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
environment:
|
|
<<: *api-environment
|
|
MIGRATE_ONLY: "true"
|
|
|
|
api:
|
|
<<: *api-build
|
|
restart: unless-stopped
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
rustfs:
|
|
condition: service_healthy
|
|
rabbitmq:
|
|
condition: service_healthy
|
|
s3-init:
|
|
condition: service_completed_successfully
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
environment:
|
|
<<: *api-environment
|
|
RUN_MIGRATIONS: "false"
|
|
expose:
|
|
- "8080"
|
|
healthcheck:
|
|
test:
|
|
["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8080/health/ready"]
|
|
interval: 2s
|
|
timeout: 2s
|
|
retries: 10
|
|
start_period: 10s
|
|
stop_grace_period: 15s
|
|
deploy:
|
|
replicas: ${API_REPLICAS:-2}
|
|
resources:
|
|
limits:
|
|
cpus: "${API_CPU_LIMIT:-0.50}"
|
|
memory: "${API_MEM_LIMIT:-512M}"
|
|
|
|
worker:
|
|
<<: *api-build
|
|
restart: unless-stopped
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
rustfs:
|
|
condition: service_healthy
|
|
rabbitmq:
|
|
condition: service_healthy
|
|
s3-init:
|
|
condition: service_completed_successfully
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
environment:
|
|
<<: *api-environment
|
|
RUN_MIGRATIONS: "false"
|
|
WORKER_ONLY: "true"
|
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
|
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
|
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
|
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
|
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
|
|
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-30}
|
|
LEASE_SWEEP_SECONDS: ${LEASE_SWEEP_SECONDS:-15}
|
|
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
|
WORK_TEMP_DIR: /tmp/messenger-work
|
|
# Test hooks, off by default. They make the grade-4 failure windows
|
|
# deterministic; see evidence/ and REPORT.md.
|
|
WORKER_FAULT_MODE: ${WORKER_FAULT_MODE:-none}
|
|
WORKER_CRASH_AFTER_EFFECT: ${WORKER_CRASH_AFTER_EFFECT:-false}
|
|
WORKER_CRASH_AFTER_CLAIM: ${WORKER_CRASH_AFTER_CLAIM:-false}
|
|
entrypoint: ["/usr/local/bin/worker"]
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pgrep -f /usr/local/bin/worker >/dev/null"]
|
|
interval: 3s
|
|
timeout: 2s
|
|
retries: 10
|
|
start_period: 5s
|
|
stop_grace_period: 20s
|
|
deploy:
|
|
replicas: ${WORKER_REPLICAS:-1}
|
|
resources:
|
|
limits:
|
|
cpus: "${WORKER_CPU_LIMIT:-1.0}"
|
|
memory: "${WORKER_MEM_LIMIT:-1G}"
|
|
|
|
# AGENT: the reconciler uses the same scoped credential as the API. It never
|
|
# touches ready attachments; it only removes stale pending rows/objects and
|
|
# objects with no metadata row at all.
|
|
reconciler:
|
|
<<: *api-build
|
|
restart: unless-stopped
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
rustfs:
|
|
condition: service_healthy
|
|
s3-init:
|
|
condition: service_completed_successfully
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
environment:
|
|
<<: *api-environment
|
|
RUN_MIGRATIONS: "false"
|
|
RECONCILE_ONLY: "true"
|
|
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900}
|
|
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-300}
|
|
entrypoint: ["/usr/local/bin/reconciler"]
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${RECONCILER_CPU_LIMIT:-0.25}"
|
|
memory: "${RECONCILER_MEM_LIMIT:-128M}"
|
|
|
|
haproxy:
|
|
image: haproxy:3.2.6-alpine
|
|
restart: unless-stopped
|
|
depends_on:
|
|
api:
|
|
condition: service_healthy
|
|
ports:
|
|
- "${LB_PORT:-8080}:8080"
|
|
- "${HAPROXY_STATS_PORT:-8404}:8404"
|
|
volumes:
|
|
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
healthcheck:
|
|
test:
|
|
["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"]
|
|
interval: 2s
|
|
timeout: 2s
|
|
retries: 10
|
|
start_period: 5s
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${HAPROXY_CPU_LIMIT:-0.50}"
|
|
memory: "${HAPROXY_MEM_LIMIT:-128M}"
|
|
|
|
k6:
|
|
image: grafana/k6:1.3.0
|
|
profiles: ["load"]
|
|
depends_on:
|
|
haproxy:
|
|
condition: service_healthy
|
|
user: "0:0"
|
|
environment:
|
|
BASE_URL: http://haproxy:8080
|
|
VUS: ${K6_VUS:-10}
|
|
DURATION: ${K6_DURATION:-60s}
|
|
volumes:
|
|
- ./load:/scripts:ro
|
|
- ./evidence:/results
|
|
command: ["run", "/scripts/messages.js"]
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: "${K6_CPU_LIMIT:-1.0}"
|
|
memory: "${K6_MEM_LIMIT:-512M}"
|
|
|
|
volumes:
|
|
pgdata:
|
|
redisdata:
|
|
rustfsdata:
|
|
rabbitmqdata:
|