456 lines
15 KiB
YAML
456 lines
15 KiB
YAML
#file: noinspection ComposeUnknownValues
|
|
# Lab6: API x2 -> durable S3 staging + PostgreSQL outbox -> publisher
|
|
# -> RabbitMQ -> worker -> final private S3 objects + PostgreSQL metadata.
|
|
# Start: see REPORT.md, "Запуск из чистого clone".
|
|
name: messenger-lab6
|
|
|
|
x-api-env: &api-env
|
|
APP_HOST: 0.0.0.0
|
|
APP_PORT: "8080"
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
|
|
SESSION_STORE: valkey
|
|
CACHE_URL_FILE: /run/secrets/cache_url
|
|
SESSION_CACHE_TTL_SECONDS: "60"
|
|
CACHE_TIMEOUT_SECONDS: "0.25"
|
|
CACHE_MAX_CONNECTIONS: "20"
|
|
CACHE_RETRY_AFTER_SECONDS: "5"
|
|
DB_POOL_SIZE: "10"
|
|
DB_MAX_OVERFLOW: "5"
|
|
DB_POOL_TIMEOUT_SECONDS: "3"
|
|
DB_CONNECT_TIMEOUT_SECONDS: "3"
|
|
DB_STATEMENT_TIMEOUT_MS: "5000"
|
|
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
|
|
SHUTDOWN_DRAIN_SECONDS: "3"
|
|
SHUTDOWN_TIMEOUT_SECONDS: "8"
|
|
# Lab4: S3 with the API's own restricted credentials (not admin).
|
|
S3_ENDPOINT: http://s3:9000
|
|
S3_REGION: us-east-1
|
|
S3_BUCKET: messenger-attachments
|
|
S3_OBJECT_PREFIX: originals/
|
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
|
S3_CONNECT_TIMEOUT_SECONDS: "2"
|
|
S3_READ_TIMEOUT_SECONDS: "5"
|
|
S3_OPERATION_TIMEOUT_SECONDS: "6"
|
|
S3_PUT_MIN_BYTES_PER_SECOND: "1048576"
|
|
MAX_UPLOAD_BYTES: "10485760"
|
|
MAX_IMAGE_PIXELS: "40000000"
|
|
UPLOAD_SPOOL_MEMORY_BYTES: "1048576"
|
|
UPLOAD_TIMEOUT_SECONDS: "60"
|
|
MAX_CONCURRENT_UPLOADS: "8"
|
|
UPLOAD_SLOT_WAIT_SECONDS: "10"
|
|
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
|
|
JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}"
|
|
JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}"
|
|
JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-30}"
|
|
JOB_TOTAL_TIMEOUT_SECONDS: "${JOB_TOTAL_TIMEOUT_SECONDS:-90}"
|
|
WORKER_MEMORY_MB: "${WORKER_MEMORY_MB:-768}"
|
|
JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-120}"
|
|
RETRY_BASE_SECONDS: "${RETRY_BASE_SECONDS:-1}"
|
|
STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-604800}"
|
|
PUBLISHER_FAULT: "${PUBLISHER_FAULT:-}"
|
|
WORKER_FAULT: "${WORKER_FAULT:-}"
|
|
# API fault-injection is disabled during normal operation.
|
|
FAULT_INJECTION: "${FAULT_INJECTION:-}"
|
|
OTEL_SDK_DISABLED: "${OTEL_SDK_DISABLED:-false}"
|
|
OTEL_SERVICE_NAME: messenger-api
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
|
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
|
OTEL_RESOURCE_ATTRIBUTES: service.version=1.6.0,deployment.environment.name=lab
|
|
OTEL_METRIC_EXPORT_INTERVAL: "5000"
|
|
|
|
services:
|
|
db:
|
|
image: postgres:16.4-alpine
|
|
environment:
|
|
POSTGRES_DB: messenger
|
|
POSTGRES_USER: messenger
|
|
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
|
secrets: [postgres_password]
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 20
|
|
start_period: 5s
|
|
networks: [backend]
|
|
stop_grace_period: 30s
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "1.0", memory: 512M }
|
|
|
|
# Session cache only (lab2). PostgreSQL stays the source of truth, so the
|
|
# cache is intentionally not persisted. It is shared by all API replicas:
|
|
# a logout on one replica writes the "revoked" tombstone seen by all.
|
|
valkey:
|
|
image: valkey/valkey:8.0.1-alpine
|
|
command:
|
|
- sh
|
|
- -c
|
|
- >-
|
|
exec valkey-server
|
|
--requirepass "$$(cat /run/secrets/valkey_password)"
|
|
--save ""
|
|
--appendonly no
|
|
--maxmemory 64mb
|
|
--maxmemory-policy allkeys-lru
|
|
secrets: [valkey_password]
|
|
healthcheck:
|
|
test:
|
|
- CMD-SHELL
|
|
- valkey-cli -a "$$(cat /run/secrets/valkey_password)" --no-auth-warning ping | grep -q PONG
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 20
|
|
networks: [backend]
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.25", memory: 128M }
|
|
|
|
# S3: SeaweedFS all-in-one. No "ports": reachable only from the internal
|
|
# backend network; its admin web UIs are bound to 127.0.0.1 inside the
|
|
# container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume.
|
|
s3:
|
|
image: chrislusf/seaweedfs:3.97
|
|
entrypoint: ["/bin/sh", "/opt/lab/start.sh"]
|
|
environment:
|
|
# Soft limit of the Go heap: without it the GC lets SeaweedFS grow to
|
|
# the container limit under parallel 10 MiB PUTs (measured, REPORT.md).
|
|
GOMEMLIMIT: 600MiB
|
|
volumes:
|
|
- s3_data:/data
|
|
- ./infra/seaweedfs:/opt/lab:ro
|
|
secrets:
|
|
- s3_admin_access_key_id
|
|
- s3_admin_secret_access_key
|
|
- s3_access_key_id
|
|
- s3_secret_access_key
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"]
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 30
|
|
start_period: 60s
|
|
networks: [backend]
|
|
stop_grace_period: 30s
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
# 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed
|
|
# SeaweedFS to the limit and the kernel killed it (REPORT.md).
|
|
limits: { cpus: "1.0", memory: 1G }
|
|
|
|
# Idempotent bucket bootstrap + self-checks (private bucket, least
|
|
# privilege of the API key). The only container with the S3 admin key.
|
|
s3-init:
|
|
build: .
|
|
image: messenger-lab6-api:${APP_VERSION:-dev}
|
|
command: ["python", "scripts/s3_bootstrap.py"]
|
|
environment:
|
|
S3_ENDPOINT: http://s3:9000
|
|
S3_BUCKET: messenger-attachments
|
|
S3_OBJECT_PREFIX: originals/
|
|
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
|
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
|
secrets:
|
|
- s3_admin_access_key_id
|
|
- s3_admin_secret_access_key
|
|
- s3_access_key_id
|
|
- s3_secret_access_key
|
|
depends_on:
|
|
s3: { condition: service_healthy }
|
|
networks: [backend]
|
|
restart: "no"
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 128M }
|
|
|
|
# Runs the migrations once, before any API replica starts (no race of
|
|
# replicas). A repeated run only prints SKIP.
|
|
migrate:
|
|
build: .
|
|
image: messenger-lab6-api:${APP_VERSION:-dev}
|
|
command: ["python", "scripts/migrate.py"]
|
|
environment:
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
secrets: [database_url]
|
|
depends_on:
|
|
db: { condition: service_healthy }
|
|
networks: [backend]
|
|
restart: "no"
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 128M }
|
|
|
|
api:
|
|
build: .
|
|
image: messenger-lab6-api:${APP_VERSION:-dev}
|
|
environment: *api-env
|
|
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
|
# No "ports": replicas are reachable only through the LB.
|
|
# No "container_name": the service is scaled by deploy.replicas / --scale.
|
|
networks: [edge, backend]
|
|
depends_on:
|
|
db: { condition: service_healthy }
|
|
valkey: { condition: service_healthy }
|
|
migrate: { condition: service_completed_successfully }
|
|
s3-init: { condition: service_completed_successfully }
|
|
healthcheck:
|
|
test:
|
|
- CMD
|
|
- python
|
|
- -c
|
|
- "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 2
|
|
start_period: 5s
|
|
stop_signal: SIGTERM
|
|
# > SHUTDOWN_DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (3 + 8 s).
|
|
stop_grace_period: 15s
|
|
restart: unless-stopped
|
|
deploy:
|
|
replicas: ${API_REPLICAS:-2}
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 512M }
|
|
|
|
# A stable nodename keeps durable queues in the same RabbitMQ database
|
|
# even after the container is re-created. No AMQP/management port is public.
|
|
broker:
|
|
image: rabbitmq:3.13.7-alpine
|
|
hostname: broker
|
|
entrypoint: ["/bin/sh", "-ec"]
|
|
command:
|
|
- 'export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"; exec docker-entrypoint.sh rabbitmq-server'
|
|
environment:
|
|
RABBITMQ_DEFAULT_USER: messenger
|
|
RABBITMQ_DEFAULT_VHOST: /
|
|
# Bound Erlang schedulers to this small container, rather than the
|
|
# host's core count; health-check CLI VMs need only one scheduler.
|
|
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
|
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
|
|
secrets: [broker_password]
|
|
volumes:
|
|
- broker_data:/var/lib/rabbitmq
|
|
networks: [backend]
|
|
healthcheck:
|
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
|
interval: 3s
|
|
timeout: 5s
|
|
retries: 20
|
|
start_period: 15s
|
|
stop_grace_period: 20s
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 512M }
|
|
|
|
publisher:
|
|
build: .
|
|
image: messenger-lab6-api:${APP_VERSION:-dev}
|
|
command: ["python", "-m", "app.jobs"]
|
|
environment:
|
|
<<: *api-env
|
|
OTEL_SERVICE_NAME: messenger-publisher
|
|
BROKER_URL_FILE: /run/secrets/broker_url
|
|
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
|
networks: [backend]
|
|
depends_on:
|
|
broker: { condition: service_healthy }
|
|
migrate: { condition: service_completed_successfully }
|
|
s3-init: { condition: service_completed_successfully }
|
|
restart: unless-stopped
|
|
stop_grace_period: 10s
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 256M }
|
|
|
|
worker:
|
|
build: .
|
|
image: messenger-lab6-api:${APP_VERSION:-dev}
|
|
command: ["python", "-m", "app.worker"]
|
|
environment:
|
|
<<: *api-env
|
|
OTEL_SERVICE_NAME: messenger-worker
|
|
BROKER_URL_FILE: /run/secrets/broker_url
|
|
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
|
networks: [backend]
|
|
depends_on:
|
|
broker: { condition: service_healthy }
|
|
migrate: { condition: service_completed_successfully }
|
|
s3-init: { condition: service_completed_successfully }
|
|
restart: unless-stopped
|
|
stop_grace_period: 10s
|
|
# One active job and one reserved delivery; image subprocess has its own
|
|
# address-space/time limits in addition to this container memory limit.
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "1.0", memory: 1G }
|
|
|
|
lb:
|
|
image: haproxy:3.0.11-alpine
|
|
volumes:
|
|
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
ports:
|
|
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
|
# Backend state page for the operator, only on the host loopback.
|
|
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
|
networks: [edge]
|
|
depends_on:
|
|
api: { condition: service_healthy }
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
|
|
interval: 5s
|
|
timeout: 2s
|
|
retries: 3
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 128M }
|
|
|
|
# Telemetry is deliberately absent from API depends_on/readiness.
|
|
# No OTLP ports on the host: application traffic stays in backend.
|
|
otel-collector:
|
|
image: otel/opentelemetry-collector-contrib:0.111.0
|
|
command: ["--config=/etc/otelcol/collector.yaml"]
|
|
volumes:
|
|
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
|
networks: [backend]
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 256M }
|
|
|
|
prometheus:
|
|
image: prom/prometheus:v2.55.0
|
|
command:
|
|
- --config.file=/etc/prometheus/prometheus.yaml
|
|
- --storage.tsdb.path=/prometheus
|
|
- --storage.tsdb.retention.time=7d
|
|
- --storage.tsdb.retention.size=512MB
|
|
volumes:
|
|
- ./infra/prometheus:/etc/prometheus:ro
|
|
- prometheus_data:/prometheus
|
|
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"]
|
|
networks: [backend, observability]
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 512M }
|
|
|
|
loki:
|
|
image: grafana/loki:3.2.1
|
|
command: ["-config.file=/etc/loki/loki.yaml"]
|
|
volumes:
|
|
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
|
- loki_data:/loki
|
|
ports: ["127.0.0.1:${LOKI_PORT:-3100}:3100"]
|
|
networks: [backend, observability]
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 384M }
|
|
|
|
tempo:
|
|
image: grafana/tempo:2.6.1
|
|
environment:
|
|
# Bound the Go heap while searching many retained blocks on this small stand.
|
|
GOMEMLIMIT: 256MiB
|
|
# A new named volume needs write access for the local evaluation backend.
|
|
user: "0:0"
|
|
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
|
volumes:
|
|
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
|
- tempo_data:/var/tempo
|
|
ports: ["127.0.0.1:${TEMPO_PORT:-3200}:3200"]
|
|
networks: [backend, observability]
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "1.0", memory: 512M }
|
|
|
|
grafana:
|
|
image: grafana/grafana:11.2.2
|
|
environment:
|
|
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
|
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
|
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
|
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
|
volumes:
|
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
|
- grafana_data:/var/lib/grafana
|
|
ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"]
|
|
networks: [backend, observability]
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits: { cpus: "0.5", memory: 256M }
|
|
|
|
# Runs the public and own checks in Linux (needed on Windows: the hooks are
|
|
# POSIX .sh scripts that call the docker CLI). Profile "tools": not started
|
|
# by a plain "up". The Docker socket grants control over the Docker host —
|
|
# use it only on the lab stand.
|
|
tester:
|
|
profiles: [tools]
|
|
build:
|
|
context: .
|
|
dockerfile: student-tests/Dockerfile.tester
|
|
working_dir: /work
|
|
environment:
|
|
BASE_URL: http://lb:8080
|
|
COMPOSE_PROJECT: messenger-lab6
|
|
S3_ENDPOINT: http://s3:9000
|
|
volumes:
|
|
- ./:/work
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
# backend: only for the "private bucket from outside the API" check.
|
|
networks: [edge, backend]
|
|
|
|
networks:
|
|
observability: {}
|
|
edge: {}
|
|
# PostgreSQL, Valkey and S3: no route outside, the LB is not attached.
|
|
backend:
|
|
internal: true
|
|
|
|
volumes:
|
|
postgres_data: {}
|
|
s3_data: {}
|
|
broker_data: {}
|
|
prometheus_data: {}
|
|
loki_data: {}
|
|
tempo_data: {}
|
|
grafana_data: {}
|
|
|
|
secrets:
|
|
postgres_password:
|
|
file: ./infra/secrets/postgres_password.txt
|
|
database_url:
|
|
file: ./infra/secrets/database_url.txt
|
|
valkey_password:
|
|
file: ./infra/secrets/valkey_password.txt
|
|
cache_url:
|
|
file: ./infra/secrets/cache_url.txt
|
|
s3_admin_access_key_id:
|
|
file: ./infra/secrets/s3_admin_access_key_id.txt
|
|
s3_admin_secret_access_key:
|
|
file: ./infra/secrets/s3_admin_secret_access_key.txt
|
|
s3_access_key_id:
|
|
file: ./infra/secrets/s3_access_key_id.txt
|
|
s3_secret_access_key:
|
|
file: ./infra/secrets/s3_secret_access_key.txt
|
|
broker_password:
|
|
file: ./infra/secrets/broker_password.txt
|
|
broker_url:
|
|
file: ./infra/secrets/broker_url.txt
|