Files
mtusi2026lab5/compose.yaml
T

346 lines
10 KiB
YAML

# Local stand: HAProxy in front of interchangeable API replicas over shared
# PostgreSQL, Valkey and S3. Only the balancer is published, on loopback; the data
# network has no way out. The load-test knobs are in compose.bench.yaml.
name: lab5
services:
lb:
image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa
depends_on:
api:
condition: service_healthy
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "${PUBLISH:-127.0.0.1:8080}:8080"
- "${STATS_PUBLISH:-127.0.0.1:8404}:8404"
networks:
- edge
healthcheck:
# Through the balancer: `up --wait` returns once a replica is reachable.
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 3
start_period: 20s
start_interval: 1s
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
api:
build: .
image: mtusi-messenger:lab5
env_file:
- path: .env
required: false
environment:
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_STORE_URL_FILE: /run/secrets/session_store_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
# The background reconciler uses a separate identity with List rights.
S3_RECONCILE_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id
S3_RECONCILE_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key
# One decoder per replica. Progressive JPEG retains coefficient buffers;
# a wide RGBA64 PNG also holds two scanlines beside the full pixel buffer.
GOMEMLIMIT: 1536MiB
secrets:
- database_url
- session_store_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
- s3_reconcile_access_key_id
- s3_reconcile_secret_access_key
networks:
- edge
- data
depends_on:
migrate:
condition: service_completed_successfully
postgres:
condition: service_healthy
valkey:
condition: service_healthy
s3-init:
condition: service_completed_successfully
# Binary deadline 30s plus compensation, shutdown and drain margin.
stop_grace_period: 45s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "1.0"
memory: 2G
broker:
image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
hostname: broker
secrets:
- source: rabbitmq_config
target: /etc/rabbitmq/conf.d/90-lab.conf
volumes:
- rabbitdata:/var/lib/rabbitmq
networks: [data]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 12
deploy:
resources:
limits:
cpus: "1"
memory: 512M
worker:
build:
context: .
target: worker
image: mtusi-messenger-worker:lab5
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "2"
memory: 2G
migrate:
image: mtusi-messenger:lab5
entrypoint: ["/usr/local/bin/migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
networks:
- data
depends_on:
postgres:
condition: service_healthy
restart: "no"
healthcheck:
# The image's check probes the API port, which migrate never opens.
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
postgres:
image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873
# 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql.
command: ["postgres", "-c", "max_connections=150"]
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
# PostgreSQL 18 keeps PGDATA in a versioned directory below this one.
volumes:
- pgdata:/var/lib/postgresql
networks:
- data
healthcheck:
# Over TCP: the first initialisation runs a temporary socket-only server.
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
stop_grace_period: 30s
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
valkey:
image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
secrets:
- valkey_auth
volumes:
- ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro
- valkeydata:/data
networks:
- data
healthcheck:
# valkey-cli exits 0 even on NOAUTH, so the reply itself is checked.
test: ["CMD-SHELL", "VALKEYCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" valkey-cli ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 10
start_period: 5s
stop_grace_period: 10s
deploy:
resources:
limits:
cpus: "0.5"
# maxmemory 256mb (valkey.conf) plus the server's own overhead.
memory: 320M
s3:
image: chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0
command: ["server", "-options=/etc/seaweedfs/lab.options"]
environment:
GOMEMLIMIT: 460MiB
secrets:
- s3_identities
volumes:
- ./infra/s3/seaweedfs.options:/etc/seaweedfs/lab.options:ro
# Both volume data and the image's /data/filerldb2 metadata live here.
- s3data:/data
networks:
- data
healthcheck:
# A reachable anonymous request must be refused, including before init.
test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q ' 403 '"]
interval: 5s
timeout: 3s
retries: 12
start_period: 15s
start_interval: 1s
stop_grace_period: 30s
pids_limit: 256
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
pids: 256
s3-init:
build: .
image: mtusi-messenger:lab5
entrypoint: ["/usr/local/bin/s3init"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
GOMEMLIMIT: 100MiB
secrets:
- source: s3_admin_access_key_id
target: s3_access_key_id
- source: s3_admin_secret_access_key
target: s3_secret_access_key
networks:
- data
depends_on:
s3:
condition: service_healthy
restart: "no"
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
reconcile:
profiles: ["recon"]
build: .
image: mtusi-messenger:lab5
entrypoint: ["/usr/local/bin/reconcile"]
environment:
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
GOMEMLIMIT: 100MiB
secrets:
- database_url
- source: s3_reconcile_access_key_id
target: s3_access_key_id
- source: s3_reconcile_secret_access_key
target: s3_secret_access_key
networks:
- data
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: "no"
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
volumes:
rabbitdata:
pgdata:
valkeydata:
s3data:
networks:
edge: {}
data:
internal: true
secrets:
broker_url:
file: ./secrets/broker_url
rabbitmq_config:
file: ./secrets/rabbitmq.conf
database_url:
file: ./secrets/database_url
session_store_url:
file: ./secrets/session_store_url
postgres_password:
file: ./secrets/postgres_password
valkey_auth:
file: ./secrets/valkey_auth.conf
s3_identities:
file: ./secrets/s3_identities.json
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_reconcile_access_key_id:
file: ./secrets/s3_reconcile_access_key_id
s3_reconcile_secret_access_key:
file: ./secrets/s3_reconcile_secret_access_key