Files
mtusi2026lab7/compose.yaml
T
HellsTomatoandCopilot 2ef0299e34
Skeleton contracts / contracts (pull_request) Canceled after 0s
Complete lab7 grade 4 implementation
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-06 15:35:53 +03:00

718 lines
23 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Compose-стенд lab7: HTTPS nginx → mTLS API-реплики; internal TLS identities are issued locally.
# телеметрия: API/worker (OTel Java agent) → OTLP → otel-collector → Prometheus / Loki / Tempo → Grafana.
# Перед первым запуском: python3 scripts/init_secrets.py — секреты появятся в secrets/ (вне Git).
# Общие переменные; FAULT_INJECTION=1 включают только эксперименты (tests/pipeline_experiment.py).
x-env: &env
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/secrets/ca.crt
SPRING_DATASOURCE_USERNAME: messenger
SPRING_DATA_REDIS_HOST: redis
SPRING_DATA_REDIS_PORT: 6379
SPRING_DATA_REDIS_SSL_ENABLED: "true"
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: https://s3-tls:8443
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
JOB_MAX_ATTEMPTS: 5
PIPELINE_VERSION: 1
FAULT_INJECTION: ${FAULT_INJECTION:-0}
BAO_ADDR: https://openbao:8200
BAO_ENABLED: ${BAO_ENABLED:-false}
BAO_ROLE: api
BAO_ROLE_ID_FILE: /run/secrets/bao-role-id
BAO_SECRET_ID_FILE: /run/secrets/bao-secret-id
# Телеметрия API и worker: OTel Java agent, экспорт OTLP/HTTP в Collector раз в 10 с (метрики).
x-otel: &otel
JAVA_TOOL_OPTIONS: -javaagent:/app/opentelemetry-javaagent.jar -Djavax.net.ssl.trustStore=/run/secrets/broker-ca.p12 -Djavax.net.ssl.trustStorePassword=changeit
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-gateway:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca.crt
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel.crt
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel.key
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=7.0.0
OTEL_METRIC_EXPORT_INTERVAL: 10000
# key-value из SLF4J (event=…) становятся атрибутами записи лога.
OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_CAPTURE_KEY_VALUE_PAIR_ATTRIBUTES: "true"
# Общая часть API-реплик, worker, migration и инструментов: один образ, разные режимы запуска.
x-api: &api
build: .
image: messenger-api:lab7
environment:
<<: [*env, *otel]
API_MTLS_ENABLED: "true"
# Секреты читаются Spring как свойства: имя файла в /run/secrets = имя свойства (configtree).
secrets:
- source: pki_ca
target: ca.crt
- source: pki_api_cert
target: api.crt
- source: pki_api_key
target: api.key
- source: pki_lb_cert
target: lb.crt
- source: pki_lb_key
target: lb.key
- source: db_password
target: spring.datasource.password
- source: redis_password
target: spring.data.redis.password
- source: s3_api_access_key
target: s3.access-key
- source: s3_api_secret_key
target: s3.secret-key
- source: broker_api_url
target: broker.url
- source: pki_api_broker_p12
target: broker-client.p12
- source: pki_broker_ca_p12
target: broker-ca.p12
- source: pki_api_otel_cert
target: otel.crt
- source: pki_api_otel_key
target: otel.key
- source: bao_api_role_id
target: bao-role-id
- source: bao_api_secret_id
target: bao-secret-id
- source: pki_redis_cert
target: redis.crt
- source: pki_redis_key
target: redis.key
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
s3-init:
condition: service_completed_successfully
migration:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "--no-check-certificate", "--certificate=/run/secrets/lb.crt", "--private-key=/run/secrets/lb.key", "https://localhost:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 3
# Запас на одновременный старт нескольких JVM при лимите 1 CPU; healthy наступает с первой успешной проверки.
start_period: 90s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits:
cpus: '1.0'
memory: 512M
reservations:
cpus: '0.5'
memory: 256M
services:
# Nginx балансирует запросы между API-репликами; единственный опубликованный порт стенда.
nginx:
image: nginx:1.28.0-alpine
ports:
- "8443:8443"
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro
secrets:
- source: pki_ingress_cert
target: ingress.crt
- source: pki_ingress_key
target: ingress.key
- source: pki_lb_cert
target: lb.crt
- source: pki_lb_key
target: lb.key
- source: pki_ca
target: ca.crt
# service_started, а не healthy: адреса реплик nginx перечитывает сам (resolve), неготовую реплику
# пропускает (proxy_next_upstream), а `up -d` не ждёт прогрева двух JVM с агентом OTel — перезапуск
# стенда укладывается в 90 с hook из tests/resilience.py.
depends_on:
api-1:
condition: service_started
api-2:
condition: service_started
restart: unless-stopped
read_only: true
security_opt: ["no-new-privileges:true"]
tmpfs: [/var/cache/nginx, /var/run]
deploy:
resources:
limits:
cpus: '0.5'
memory: 128M
# Две взаимозаменяемые API-реплики без локального состояния.
api-1: *api
api-2: *api
# Worker: отдельный процесс (режим worker) с собственными ключами S3 и пользователем брокера.
worker:
<<: *api
command: ["worker"]
environment:
<<: [*env, *otel]
OTEL_SERVICE_NAME: messenger-worker
BAO_ROLE: worker
WORKER_CONCURRENCY: 2
JOB_TIMEOUT_SECONDS: 60
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
STAGING_RETENTION_SECONDS: 86400
# Куча до 65% лимита: декодирование одного изображения ограничено ~48 MiB растра.
JAVA_TOOL_OPTIONS: -javaagent:/app/opentelemetry-javaagent.jar -XX:MaxRAMPercentage=65 -Djavax.net.ssl.trustStore=/run/secrets/broker-ca.p12 -Djavax.net.ssl.trustStorePassword=changeit
secrets:
- source: pki_ca
target: ca.crt
- source: db_password
target: spring.datasource.password
- source: s3_worker_access_key
target: s3.access-key
- source: s3_worker_secret_key
target: s3.secret-key
- source: broker_worker_url
target: broker.url
- source: pki_worker_broker_p12
target: broker-client.p12
- source: pki_broker_ca_p12
target: broker-ca.p12
- source: pki_worker_otel_cert
target: otel.crt
- source: pki_worker_otel_key
target: otel.key
- source: bao_worker_role_id
target: bao-role-id
- source: bao_worker_secret_id
target: bao-secret-id
- source: pki_redis_cert
target: redis.crt
- source: pki_redis_key
target: redis.key
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
s3-init:
condition: service_completed_successfully
migration:
condition: service_completed_successfully
# Живой = подключён к брокеру (файл обновляется каждые 2 с).
healthcheck:
test: ["CMD-SHELL", "test -n \"$$(find /tmp/worker-alive -newermt '-15 seconds' 2>/dev/null)\""]
interval: 5s
timeout: 3s
retries: 3
start_period: 90s
# Операторские команды заданий: docker compose run --rm jobctl status | replay | reprocess | redispatch.
jobctl:
<<: *api
environment:
<<: *env
JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/run/secrets/broker-ca.p12 -Djavax.net.ssl.trustStorePassword=changeit
entrypoint: ["java", "-Dfile.encoding=UTF-8", "-jar", "/app/messenger.jar", "jobctl"]
command: []
profiles: ["tools"]
secrets:
- source: pki_ca
target: ca.crt
- source: pki_broker_ca_p12
target: broker-ca.p12
- source: db_password
target: spring.datasource.password
- source: s3_api_access_key
target: s3.access-key
- source: s3_api_secret_key
target: s3.secret-key
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
restart: "no"
# Однократные миграции Flyway до запуска реплик; нужен только пароль БД.
migration:
<<: *api
environment: *env
command: ["migrate"]
secrets:
- source: pki_ca
target: ca.crt
- source: db_password
target: spring.datasource.password
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
restart: "no"
# Сверка БД и S3 по требованию: docker compose run --rm reconciler [--apply] [--grace-seconds=N].
reconciler:
<<: *api
environment:
<<: *env
JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/run/secrets/broker-ca.p12 -Djavax.net.ssl.trustStorePassword=changeit
# Режим в entrypoint: аргументы docker compose run дописываются к нему.
entrypoint: ["java", "-Dfile.encoding=UTF-8", "-jar", "/app/messenger.jar", "reconcile"]
command: []
profiles: ["tools"]
secrets:
- source: pki_ca
target: ca.crt
- source: pki_broker_ca_p12
target: broker-ca.p12
- source: db_password
target: spring.datasource.password
- source: s3_api_access_key
target: s3.access-key
- source: s3_api_secret_key
target: s3.secret-key
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
healthcheck:
disable: true
restart: "no"
# RabbitMQ: durable quorum-очередь attachments.process + DLQ. Пользователи, права и топология —
# из definitions-файла (secrets/), management UI не включён, порт наружу не публикуется.
rabbitmq:
image: rabbitmq:4.0.5-alpine
# Имя узла rabbit@rabbitmq задаёт каталог данных в volume: без фиксированного hostname
# пересозданный контейнер не увидел бы прежние очереди и сообщения.
hostname: rabbitmq
volumes:
- rabbitmqdata:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab.conf:ro
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
secrets:
- source: rabbitmq_definitions
target: rabbitmq_definitions.json
- source: pki_ca
target: ca.crt
- source: pki_broker_cert
target: broker.crt
- source: pki_broker_key
target: broker.key
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 10s
timeout: 10s
retries: 12
start_period: 20s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits:
cpus: '1.0'
memory: 512M
# Collector: приём OTLP, memory_limiter/batch, ограниченные очереди экспорта (infra/otel/collector.yaml).
otel-collector:
image: otel/opentelemetry-collector-contrib:0.115.1
command: ["--config=/etc/otelcol/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
depends_on:
prometheus:
condition: service_healthy
loki:
condition: service_healthy
tempo:
condition: service_started
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
# TLS/mTLS identity gate in front of the Collector. The allow-list prevents another
# certificate signed by the lab CA from becoming an OTel client.
otel-gateway:
image: nginx:1.28.0-alpine
volumes:
- ./infra/otel/gateway.conf:/etc/nginx/conf.d/default.conf:ro
secrets:
- source: pki_ca
target: ca.crt
- source: pki_collector_cert
target: collector.crt
- source: pki_collector_key
target: collector.key
depends_on:
otel-collector:
condition: service_started
read_only: true
security_opt: ["no-new-privileges:true"]
tmpfs: [/var/cache/nginx, /var/run]
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.25'
memory: 128M
# OpenBao is deliberately internal and persistent; initialization/unseal remains operator-only.
openbao:
image: openbao/openbao:2.1.0
command: ["server", "-config=/openbao/config.hcl"]
user: "0:0"
volumes:
- bao-data:/openbao/data
- ./infra/openbao/config.hcl:/openbao/config.hcl:ro
secrets:
- source: pki_ca
target: ca.crt
- source: pki_openbao_cert
target: openbao.crt
- source: pki_openbao_key
target: openbao.key
cap_add: ["IPC_LOCK"]
security_opt: ["no-new-privileges:true"]
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
# Метрики (TSDB в volume, 7 дней), приём OTLP от Collector.
prometheus:
image: prom/prometheus:v3.0.1
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.enable-otlp-receiver"]
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
- promdata:/prometheus
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
# Логи (structured metadata с trace_id/span_id), файловое хранилище в volume.
loki:
image: grafana/loki:3.3.2
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:3100/ready"]
interval: 5s
timeout: 3s
retries: 30
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
# Трассы (локальные блоки в volume, 7 дней).
tempo:
image: grafana/tempo:2.6.1
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
# Grafana: только интерфейс и запросы; datasources и dashboard — provisioning из Git.
grafana:
image: grafana/grafana:11.4.0
ports:
- "3000:3000"
environment:
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
secrets:
- grafana_admin_password
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- grafanadata:/var/lib/grafana
depends_on:
- prometheus
- loki
- tempo
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
# PostgreSQL — источник истины: пользователи, чаты, сообщения, метаданные вложений.
postgres:
image: postgres:16.4
entrypoint: ["/bin/sh", "/usr/local/bin/lab-entrypoint.sh"]
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
command: ["postgres", "-c", "ssl=on", "-c", "ssl_cert_file=/var/lib/postgresql/data/server.crt", "-c", "ssl_key_file=/var/lib/postgresql/data/server.key", "-c", "ssl_ca_file=/var/lib/postgresql/data/ca.crt"]
secrets:
- db_password
- source: pki_ca
target: ca.crt
- source: pki_postgres_cert
target: postgres.crt
- source: pki_postgres_key
target: postgres.key
volumes:
- pgdata:/var/lib/postgresql/data
- ./infra/postgres/entrypoint.sh:/usr/local/bin/lab-entrypoint.sh:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 10
restart: unless-stopped
deploy:
resources:
limits:
cpus: '2.0'
memory: 1G
# Redis хранит Bearer-сессии; пароль и AOF заданы в redis.conf из secrets/.
redis:
image: redis:7.4.1
command: ["redis-server", "/run/secrets/redis.conf"]
secrets:
- source: redis_conf
target: redis.conf
- redis_password
- source: pki_ca
target: ca.crt
- source: pki_redis_cert
target: redis.crt
- source: pki_redis_key
target: redis.key
volumes:
- redisdata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli --tls --cacert /run/secrets/ca.crt ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 10
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
# SeaweedFS: master/volume/filer слушают только 127.0.0.1 внутри контейнера (их HTTP-админки
# без авторизации), в сеть стенда открыт только S3 :8333. Наружу порты не публикуются.
s3:
image: chrislusf/seaweedfs:3.99
command:
- server
- -dir=/data
- -ip=127.0.0.1
- -ip.bind=127.0.0.1
- -s3
- -s3.ip.bind=0.0.0.0
- -s3.port=8333
- -s3.config=/run/secrets/s3_config.json
- -master.volumeSizeLimitMB=1024
- -volume.max=0
# Мягкий предел кучи Go ниже лимита контейнера: без него при 10+ параллельных загрузках
# SeaweedFS упирался в 512M (tests/upload_memory.py, REPORT.md).
environment:
GOMEMLIMIT: 320MiB
secrets:
- source: s3_config
target: s3_config.json
volumes:
- s3data:/data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
deploy:
resources:
limits:
cpus: '1.0'
memory: 512M
# SeaweedFS exposes HTTP only; this sidecar provides the verified internal TLS hop.
s3-tls:
image: nginx:1.28.0-alpine
volumes:
- ./infra/s3/nginx.conf:/etc/nginx/conf.d/default.conf:ro
secrets:
- source: pki_s3_tls_cert
target: s3-tls.crt
- source: pki_s3_tls_key
target: s3-tls.key
depends_on:
s3:
condition: service_healthy
read_only: true
security_opt: ["no-new-privileges:true"]
tmpfs: [/var/cache/nginx, /var/run]
restart: unless-stopped
deploy:
resources:
limits:
cpus: '0.5'
memory: 128M
# Однократный идемпотентный bootstrap приватного bucket (admin-ключи есть только здесь и в s3).
s3-init:
image: curlimages/curl:8.11.1
entrypoint: ["/bin/sh", "/init/s3-init.sh"]
environment:
S3_ENDPOINT: https://s3-tls:8443
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
volumes:
- ./infra/s3/s3-init.sh:/init/s3-init.sh:ro
- ./secrets/pki/ca.crt:/run/secrets/ca.crt:ro
secrets:
- s3_admin_access_key
- s3_admin_secret_key
depends_on:
s3:
condition: service_healthy
restart: "no"
# Клиент S3 мимо API для проверок и внедрения сбоев: scripts/check-private-bucket.sh,
# tests/reconcile_experiment.py. Только ключи API; скрипт читается из stdin.
s3-probe:
image: curlimages/curl:8.11.1
profiles: ["tools"]
entrypoint: ["/bin/sh", "-eu", "-s"]
secrets:
- source: s3_api_access_key
target: s3.access-key
- source: s3_api_secret_key
target: s3.secret-key
restart: "no"
volumes:
promdata:
lokidata:
tempodata:
grafanadata:
rabbitmqdata:
pgdata:
redisdata:
s3data:
bao-data:
# Файлы создаёт scripts/init_secrets.py; в Git их нет.
secrets:
db_password:
file: ./secrets/db_password
redis_password:
file: ./secrets/redis_password
redis_conf:
file: ./secrets/redis.conf
s3_config:
file: ./secrets/s3_config.json
s3_api_access_key:
file: ./secrets/s3_api_access_key
s3_api_secret_key:
file: ./secrets/s3_api_secret_key
s3_admin_access_key:
file: ./secrets/s3_admin_access_key
s3_admin_secret_key:
file: ./secrets/s3_admin_secret_key
s3_worker_access_key:
file: ./secrets/s3_worker_access_key
s3_worker_secret_key:
file: ./secrets/s3_worker_secret_key
broker_api_url:
file: ./secrets/broker_api_url
broker_worker_url:
file: ./secrets/broker_worker_url
rabbitmq_definitions:
file: ./secrets/rabbitmq_definitions.json
grafana_admin_password:
file: ./secrets/grafana_admin_password
pki_ca:
file: ./secrets/pki/ca.crt
pki_ingress_cert:
file: ./secrets/pki/ingress.crt
pki_ingress_key:
file: ./secrets/pki/ingress.key
pki_lb_cert:
file: ./secrets/pki/lb.crt
pki_lb_key:
file: ./secrets/pki/lb.key
pki_api_cert:
file: ./secrets/pki/api.crt
pki_api_key:
file: ./secrets/pki/api.key
pki_broker_cert:
file: ./secrets/pki/broker.crt
pki_broker_key:
file: ./secrets/pki/broker.key
pki_api_broker_p12:
file: ./secrets/pki/api-broker.p12
pki_worker_broker_p12:
file: ./secrets/pki/worker-broker.p12
pki_broker_ca_p12:
file: ./secrets/pki/ca.p12
pki_collector_cert:
file: ./secrets/pki/collector.crt
pki_collector_key:
file: ./secrets/pki/collector.key
pki_api_otel_cert:
file: ./secrets/pki/api-otel.crt
pki_api_otel_key:
file: ./secrets/pki/api-otel.key
pki_worker_otel_cert:
file: ./secrets/pki/worker-otel.crt
pki_worker_otel_key:
file: ./secrets/pki/worker-otel.key
pki_openbao_cert:
file: ./secrets/pki/openbao.crt
pki_openbao_key:
file: ./secrets/pki/openbao.key
pki_postgres_cert:
file: ./secrets/pki/postgres.crt
pki_postgres_key:
file: ./secrets/pki/postgres.key
pki_redis_cert:
file: ./secrets/pki/redis.crt
pki_redis_key:
file: ./secrets/pki/redis.key
pki_s3_tls_cert:
file: ./secrets/pki/s3-tls.crt
pki_s3_tls_key:
file: ./secrets/pki/s3-tls.key
bao_api_role_id:
file: ./secrets/bao-api-role-id
bao_api_secret_id:
file: ./secrets/bao-api-secret-id
bao_worker_role_id:
file: ./secrets/bao-worker-role-id
bao_worker_secret_id:
file: ./secrets/bao-worker-secret-id