# Local stand: HAProxy in front of interchangeable API replicas over shared # PostgreSQL, Valkey and S3. Only the balancer is published, on loopback; the data # network has no way out. The load-test knobs are in compose.bench.yaml. name: lab5 services: lb: image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa depends_on: api: condition: service_healthy volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro ports: - "${PUBLISH:-127.0.0.1:8080}:8080" - "${STATS_PUBLISH:-127.0.0.1:8404}:8404" networks: - edge healthcheck: # Through the balancer: `up --wait` returns once a replica is reachable. test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] interval: 5s timeout: 3s retries: 3 start_period: 20s start_interval: 1s deploy: resources: limits: cpus: "0.5" memory: 128M api: build: . image: mtusi-messenger:lab5 env_file: - path: .env required: false environment: DATABASE_URL_FILE: /run/secrets/database_url SESSION_STORE_URL_FILE: /run/secrets/session_store_url BROKER_URL_FILE: /run/secrets/broker_url JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key # The background reconciler uses a separate identity with List rights. S3_RECONCILE_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconcile_access_key_id S3_RECONCILE_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconcile_secret_access_key # One decoder per replica. Progressive JPEG retains coefficient buffers; # a wide RGBA64 PNG also holds two scanlines beside the full pixel buffer. GOMEMLIMIT: 1536MiB secrets: - database_url - session_store_url - broker_url - s3_access_key_id - s3_secret_access_key - s3_reconcile_access_key_id - s3_reconcile_secret_access_key networks: - edge - data depends_on: migrate: condition: service_completed_successfully postgres: condition: service_healthy valkey: condition: service_healthy s3-init: condition: service_completed_successfully # Binary deadline 30s plus compensation, shutdown and drain margin. stop_grace_period: 45s deploy: replicas: ${API_REPLICAS:-2} resources: limits: cpus: "1.0" memory: 2G broker: image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2 hostname: broker secrets: - source: rabbitmq_config target: /etc/rabbitmq/conf.d/90-lab.conf volumes: - rabbitdata:/var/lib/rabbitmq networks: [data] healthcheck: test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] interval: 10s timeout: 5s retries: 12 deploy: resources: limits: cpus: "1" memory: 512M worker: build: context: . target: worker image: mtusi-messenger-worker:lab5 environment: DATABASE_URL_FILE: /run/secrets/database_url BROKER_URL_FILE: /run/secrets/broker_url S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] networks: [data] depends_on: migrate: condition: service_completed_successfully s3-init: condition: service_completed_successfully deploy: resources: limits: cpus: "2" memory: 2G migrate: image: mtusi-messenger:lab5 entrypoint: ["/usr/local/bin/migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url secrets: - database_url networks: - data depends_on: postgres: condition: service_healthy restart: "no" healthcheck: # The image's check probes the API port, which migrate never opens. disable: true deploy: resources: limits: cpus: "0.5" memory: 128M postgres: image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 # 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql. command: ["postgres", "-c", "max_connections=150"] environment: POSTGRES_DB: messenger POSTGRES_USER: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password secrets: - postgres_password # PostgreSQL 18 keeps PGDATA in a versioned directory below this one. volumes: - pgdata:/var/lib/postgresql networks: - data healthcheck: # Over TCP: the first initialisation runs a temporary socket-only server. test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] interval: 5s timeout: 3s retries: 10 start_period: 10s stop_grace_period: 30s deploy: resources: limits: cpus: "1.0" memory: 512M valkey: image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] secrets: - valkey_auth volumes: - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro - valkeydata:/data networks: - data healthcheck: # valkey-cli exits 0 even on NOAUTH, so the reply itself is checked. test: ["CMD-SHELL", "VALKEYCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" valkey-cli ping | grep -q PONG"] interval: 5s timeout: 3s retries: 10 start_period: 5s stop_grace_period: 10s deploy: resources: limits: cpus: "0.5" # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M s3: image: chrislusf/seaweedfs:4.53@sha256:ccad23e18639c8b98cee17e7100b95830a06077b56d8e5b006d6dcf8ec5e94a0 command: ["server", "-options=/etc/seaweedfs/lab.options"] environment: GOMEMLIMIT: 460MiB secrets: - s3_identities volumes: - ./infra/s3/seaweedfs.options:/etc/seaweedfs/lab.options:ro # Both volume data and the image's /data/filerldb2 metadata live here. - s3data:/data networks: - data healthcheck: # A reachable anonymous request must be refused, including before init. test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q ' 403 '"] interval: 5s timeout: 3s retries: 12 start_period: 15s start_interval: 1s stop_grace_period: 30s pids_limit: 256 deploy: resources: limits: cpus: "1.0" memory: 512M pids: 256 s3-init: build: . image: mtusi-messenger:lab5 entrypoint: ["/usr/local/bin/s3init"] environment: S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key GOMEMLIMIT: 100MiB secrets: - source: s3_admin_access_key_id target: s3_access_key_id - source: s3_admin_secret_access_key target: s3_secret_access_key networks: - data depends_on: s3: condition: service_healthy restart: "no" healthcheck: disable: true deploy: resources: limits: cpus: "0.5" memory: 128M reconcile: profiles: ["recon"] build: . image: mtusi-messenger:lab5 entrypoint: ["/usr/local/bin/reconcile"] environment: STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} DATABASE_URL_FILE: /run/secrets/database_url S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key GOMEMLIMIT: 100MiB secrets: - database_url - source: s3_reconcile_access_key_id target: s3_access_key_id - source: s3_reconcile_secret_access_key target: s3_secret_access_key networks: - data depends_on: migrate: condition: service_completed_successfully s3-init: condition: service_completed_successfully restart: "no" healthcheck: disable: true deploy: resources: limits: cpus: "0.5" memory: 128M volumes: rabbitdata: pgdata: valkeydata: s3data: networks: edge: {} data: internal: true secrets: broker_url: file: ./secrets/broker_url rabbitmq_config: file: ./secrets/rabbitmq.conf database_url: file: ./secrets/database_url session_store_url: file: ./secrets/session_store_url postgres_password: file: ./secrets/postgres_password valkey_auth: file: ./secrets/valkey_auth.conf s3_identities: file: ./secrets/s3_identities.json s3_admin_access_key_id: file: ./secrets/s3_admin_access_key_id s3_admin_secret_access_key: file: ./secrets/s3_admin_secret_access_key s3_access_key_id: file: ./secrets/s3_access_key_id s3_secret_access_key: file: ./secrets/s3_secret_access_key s3_reconcile_access_key_id: file: ./secrets/s3_reconcile_access_key_id s3_reconcile_secret_access_key: file: ./secrets/s3_reconcile_secret_access_key