Files
mtusi2026lab7-fork/compose.yml
T

530 lines
19 KiB
YAML

x-bao-environment: &bao-environment
BAO_ADDR: https://openbao:8200
SSL_CERT_FILE: /run/tls/ca.crt
x-secret-tmpfs: &secret-tmpfs /run/runtime-secrets:rw,noexec,nosuid,size=1m,mode=1777
name: highload-lab7
services:
redis:
cpus: 0.25
mem_limit: 128m
networks: [data]
image: redis:7.4.11-bookworm
command:
[
"redis-server",
"--port", "0",
"--tls-port", "6379",
"--tls-cert-file", "/run/tls/redis.crt",
"--tls-key-file", "/run/tls/redis.key",
"--tls-ca-cert-file", "/run/tls/ca.crt",
"--tls-auth-clients", "no",
"--aclfile", "/run/secrets/redis_acl",
"--appendonly",
"yes",
"--appendfsync",
"everysec",
"--dir",
"/data",
]
volumes:
- redis_data:/data
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/redis.crt:/run/tls/redis.crt:ro
- ./secrets/tls/redis.key:/run/tls/redis.key:ro
- ./secrets/redis_acl:/run/secrets/redis_acl:ro
- ./secrets/redis_password:/run/secrets/redis_password:ro
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=$$(cat /run/secrets/redis_password) redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger ping | grep -qx PONG"]
interval: 3s
timeout: 3s
retries: 20
db:
cpus: 1.0
mem_limit: 512m
networks: [data]
image: postgres:16.10-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/db.crt:/run/tls/db.crt:ro
- ./secrets/tls/db.key:/run/tls/db.key:ro
- ./secrets/database_api_password:/run/secrets/database_api_password:ro
- ./secrets/database_worker_password:/run/secrets/database_worker_password:ro
- ./secrets/database_dispatcher_password:/run/secrets/database_dispatcher_password:ro
- ./secrets/database_cleanup_password:/run/secrets/database_cleanup_password:ro
- ./infra/postgres-hba.conf:/run/tls/pg_hba.conf:ro
entrypoint: ["/bin/sh", "-ec", "install -o postgres -g postgres -m 600 /run/tls/db.key /tmp/db.key; exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/run/tls/db.crt -c ssl_key_file=/tmp/db.key -c hba_file=/run/tls/pg_hba.conf"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 3s
timeout: 3s
retries: 20
openbao:
image: openbao/openbao:2.4.4
user: openbao
networks: [bao, edge]
ports: ["127.0.0.1:8200:8200"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
mem_limit: 256m
cpus: 0.5
entrypoint: ["bao"]
command: ["server", "-config=/openbao/config/server.hcl"]
volumes:
- openbao_data:/openbao/file
- ./infra/openbao/server.hcl:/openbao/config/server.hcl:ro
- ./secrets/tls/openbao.crt:/run/tls/bao.crt:ro
- ./secrets/tls/openbao.key:/run/tls/bao.key:ro
migrate:
cpus: 0.5
mem_limit: 128m
networks: [data]
build: ./src
environment:
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
volumes:
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/database_tls_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
command: ["./messenger", "migrate", "up"]
depends_on:
db:
condition: service_healthy
s3:
cpus: 1.0
mem_limit: 512m
networks: [edge, data]
ports:
- "127.0.0.1:9000:9000"
build:
context: ./infra
dockerfile: minio.Dockerfile
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key
MINIO_BROWSER: "off"
MINIO_API_CORS_ALLOW_ORIGIN: ${S3_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}
volumes:
- s3_data:/data
- ./secrets/tls/s3.crt:/run/minio-certs/public.crt:ro
- ./secrets/tls/s3.key:/run/minio-certs/private.key:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
command: ["server", "/data", "--certs-dir", "/run/minio-certs", "--console-address", "127.0.0.1:9001"]
healthcheck:
test: ["CMD", "curl", "--cacert", "/run/tls/ca.crt", "-f", "https://s3:9000/minio/health/ready"]
interval: 3s
timeout: 2s
retries: 30
s3-bootstrap:
cpus: 0.5
mem_limit: 128m
networks: [data]
build: ./src
environment: &s3-environment
SSL_CERT_FILE: /run/tls/ca.crt
S3_CLEANUP_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_cleanup_secret_access_key
S3_CLEANUP_ACCESS_KEY_ID_FILE: /run/secrets/s3_cleanup_access_key_id
S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
S3_API_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
S3_API_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
S3_ENDPOINT: https://s3:9000
S3_PUBLIC_ENDPOINT: https://localhost:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
volumes:
- ./secrets/s3_cleanup_secret_access_key:/run/secrets/s3_cleanup_secret_access_key:ro
- ./secrets/s3_cleanup_access_key_id:/run/secrets/s3_cleanup_access_key_id:ro
- ./secrets/s3_worker_secret_access_key:/run/secrets/s3_worker_secret_access_key:ro
- ./secrets/s3_worker_access_key_id:/run/secrets/s3_worker_access_key_id:ro
- ./secrets/s3_api_secret_access_key:/run/secrets/s3_api_secret_access_key:ro
- ./secrets/s3_api_access_key_id:/run/secrets/s3_api_access_key_id:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
command: ["./messenger", "s3-bootstrap"]
depends_on:
s3:
condition: service_healthy
api:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
cpus: 0.5
mem_limit: 512m
networks: [proxy, data, ingest, bao]
build: ./src
deploy:
replicas: ${API_REPLICAS:-2}
stop_grace_period: 15s
environment: &api-environment
<<: [*s3-environment, *bao-environment]
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/otel/client.crt
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/otel/client.key
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
UPLOAD_TMP_DIR: /tmp/uploads
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
GOMAXPROCS: ${API_GOMAXPROCS:-1}
GOMEMLIMIT: 384MiB
BAO_ROLE: api
TLS_CA_FILE: /run/tls/ca.crt
TLS_CERT_FILE: /run/tls/api.crt
TLS_KEY_FILE: /run/tls/api.key
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
REDIS_ADDR: redis:6379
tmpfs:
- *secret-tmpfs
- /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777
volumes:
- &ca ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/api.crt:/run/tls/api.crt:ro
- ./secrets/tls/api.key:/run/tls/api.key:ro
- ./secrets/tls/bao-api.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-api.key:/run/bao/client.key:ro
- ./secrets/tls/otel-api.crt:/run/otel/client.crt:ro
- ./secrets/tls/otel-api.key:/run/otel/client.key:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
db:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
disable: true
broker:
image: rabbitmq:4.2.0-management
hostname: broker
environment:
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
cpus: 0.5
mem_limit: 512m
networks: [data]
command:
- sh
- -ec
- |
export RABBITMQ_DEFAULT_USER="$$(cat /run/secrets/broker_user)"
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"
exec docker-entrypoint.sh rabbitmq-server
volumes:
- ./infra/rabbitmq/bootstrap.sh:/bootstrap.sh:ro
- ./secrets/broker_dispatcher_password:/run/secrets/broker_dispatcher_password:ro
- ./secrets/broker_worker_password:/run/secrets/broker_worker_password:ro
- rabbitmq_data:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/broker.crt:/run/tls/broker.crt:ro
- ./secrets/tls/broker.key:/run/tls/broker.key:ro
- ./secrets/broker_user:/run/secrets/broker_user:ro
- ./secrets/broker_password:/run/secrets/broker_password:ro
healthcheck:
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running"]
interval: 5s
timeout: 10s
start_period: 20s
retries: 30
worker:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs:
- *secret-tmpfs
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
build: ./src
restart: unless-stopped
cpus: 1.0
mem_limit: 512m
networks: [data, ingest, bao]
environment:
<<: *api-environment
BAO_ROLE: worker
BROKER_URL_FILE: /run/secrets/broker_url
CACHE_URL_FILE: ""
volumes:
- *ca
- ./secrets/tls/bao-worker.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-worker.key:/run/bao/client.key:ro
- ./secrets/tls/otel-worker.crt:/run/otel/client.crt:ro
- ./secrets/tls/otel-worker.key:/run/otel/client.key:ro
command: ["./messenger", "worker"]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
dispatcher:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: [*secret-tmpfs]
build: ./src
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data, ingest, bao]
environment:
<<: *api-environment
BAO_ROLE: dispatcher
BROKER_URL_FILE: /run/secrets/broker_url
CACHE_URL_FILE: ""
S3_ACCESS_KEY_ID_FILE: ""
S3_SECRET_ACCESS_KEY_FILE: ""
volumes:
- *ca
- ./secrets/tls/bao-dispatcher.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-dispatcher.key:/run/bao/client.key:ro
- ./secrets/tls/otel-dispatcher.crt:/run/otel/client.crt:ro
- ./secrets/tls/otel-dispatcher.key:/run/otel/client.key:ro
command: ["./messenger", "dispatcher"]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
upload-cleanup:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: [*secret-tmpfs]
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data, bao]
build: ./src
command: ["./messenger", "attachments-cleanup", "--loop"]
environment:
<<: [*s3-environment, *bao-environment]
BAO_ROLE: cleanup
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
volumes:
- *ca
- ./secrets/tls/bao-cleanup.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-cleanup.key:/run/bao/client.key:ro
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
haproxy:
environment:
APP_PORT: ${APP_PORT:-8080}
API_REPLICAS: ${API_REPLICAS:-2}
cpus: 0.5
mem_limit: 128m
networks: [edge, proxy]
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:${API_HTTPS_PORT:-8443}:8443"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/ingress.pem:/run/tls/ingress.pem:ro
- ./secrets/tls/lb.pem:/run/tls/lb.pem:ro
depends_on:
api:
condition: service_started
job-admin:
profiles: ["operator"]
build: ./src
networks: [data]
cpus: 0.25
mem_limit: 128m
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
environment:
<<: *s3-environment
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
volumes:
- *ca
- ./secrets/database_tls_url:/run/secrets/database_url:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
backend-tests:
cpus: 2.0
mem_limit: 1g
networks: [data]
profiles: ["test"]
build:
context: ./src
target: builder
environment:
<<: *s3-environment
S3_PUBLIC_ENDPOINT: https://s3:9000
TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt
TEST_REDIS_ADDR: redis:6379
CACHE_URL_FILE: /run/secrets/cache_url
volumes:
- *ca
- ./secrets/cache_tls_url:/run/secrets/cache_url:ro
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
db:
condition: service_healthy
redis:
condition: service_healthy
command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."]
otel-collector:
cpus: 0.5
image: otel/opentelemetry-collector-contrib:0.120.0
networks: [telemetry, ingest]
mem_limit: 192m
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro
- ./secrets/tls/collector-exporter.crt:/run/otel/client.crt:ro
- ./secrets/tls/collector-exporter.key:/run/otel/client.key:ro
- *ca
- ./secrets/tls/otel-collector.crt:/run/otel/server.crt:ro
- ./secrets/tls/otel-collector.key:/run/otel/server.key:ro
- ./secrets/tls/telemetry-ingest-ca.crt:/run/otel/ingest-ca.crt:ro
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
prometheus:
cpus: 0.5
mem_limit: 384m
image: prom/prometheus:v3.2.1
networks: [telemetry]
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.config.file=/etc/prometheus/web.yaml"]
volumes:
- ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro
- ./secrets/tls/prometheus-scraper.crt:/run/otel/client.crt:ro
- ./secrets/tls/prometheus-scraper.key:/run/otel/client.key:ro
- *ca
- ./secrets/tls/prometheus.crt:/run/otel/server.crt:ro
- ./secrets/tls/prometheus.key:/run/otel/server.key:ro
- ./infra/otel/prometheus-web.yaml:/etc/prometheus/web.yaml:ro
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
loki:
cpus: 0.5
mem_limit: 384m
image: grafana/loki:3.4.2
networks: [telemetry]
command: ["-config.file=/etc/loki/lab.yaml"]
volumes:
- ./secrets/tls/loki-client-ca.crt:/run/otel/loki-client-ca.crt:ro
- *ca
- ./secrets/tls/loki.crt:/run/otel/server.crt:ro
- ./secrets/tls/loki.key:/run/otel/server.key:ro
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
- loki_data:/loki
tempo:
cpus: 0.5
mem_limit: 384m
image: grafana/tempo:2.7.1
networks: [telemetry]
command: ["-config.file=/etc/tempo/lab.yaml"]
volumes:
- ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro
- ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro
- *ca
- ./secrets/tls/tempo.crt:/run/otel/server.crt:ro
- ./secrets/tls/tempo.key:/run/otel/server.key:ro
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
- tempo_data:/var/tempo
grafana:
entrypoint:
- /bin/sh
- -ec
- |
export GRAFANA_READER_CERT="$$(cat /run/otel/client.crt)"
export GRAFANA_READER_KEY="$$(cat /run/otel/client.key)"
exec /run.sh
cpus: 0.5
mem_limit: 384m
image: grafana/grafana:11.5.2
networks: [telemetry, edge]
environment:
SSL_CERT_FILE: /run/tls/ca.crt
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_SERVER_PROTOCOL: https
GF_SERVER_CERT_FILE: /run/otel/server.crt
GF_SERVER_CERT_KEY: /run/otel/server.key
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
ports:
- "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000"
volumes:
- ./secrets/tls/grafana-reader.crt:/run/otel/client.crt:ro
- ./secrets/tls/grafana-reader.key:/run/otel/client.key:ro
- *ca
- ./secrets/tls/grafana.crt:/run/otel/server.crt:ro
- ./secrets/tls/grafana.key:/run/otel/server.key:ro
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ./secrets/grafana_admin_password:/run/secrets/grafana_admin_password:ro
- grafana_data:/var/lib/grafana
volumes:
openbao_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
rabbitmq_data:
postgres_data:
redis_data:
s3_data:
networks:
bao:
internal: true
edge:
proxy:
internal: true
data:
internal: true
telemetry:
internal: true
ingest:
internal: true