Files
mtusi2026lab7/compose.yaml
T

434 lines
18 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
name: messenger-lab7
x-app: &app
build: .
image: messenger-lab7-app:dev # уникальное имя: на общем Docker-host не перепутать с чужими стендами
networks: [backend]
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
tmpfs: ["/tmp:size=64m,mode=1777"]
ulimits:
core: { soft: 0, hard: 0 }
volumes: [./secrets/tls/service-ca:/run/tls:ro]
logging: &logging
driver: json-file
options: { max-size: "10m", max-file: "3" } # stdout-логи ограничены; история логов — в Loki
# OpenTelemetry SDK api и worker (src/telemetry.ts). service.name задаётся у каждого сервиса отдельно.
# OTEL_SDK_DISABLED=true — приложение без telemetry (сравнение, отладка).
x-otel-env: &otel-env
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_PROPAGATORS: tracecontext # W3C traceparent/tracestate; baggage клиентов не принимается
OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс; окна rate() от 1 минуты
OTEL_BSP_SCHEDULE_DELAY: "1000"
OTEL_BLRP_SCHEDULE_DELAY: "1000"
OTEL_NODE_EXPERIMENTAL_SDK_METRICS: "true" # otel.sdk.*: сколько spans/логов SDK обработал и отбросил
x-s3-env: &s3-env
S3_ENDPOINT: https://s3:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: messenger-attachments
NODE_EXTRA_CA_CERTS: /run/tls/ca.crt
x-jobs-env: &jobs-env
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-2000}
JOB_REPUBLISH_SECONDS: ${JOB_REPUBLISH_SECONDS:-600}
FAILPOINTS: ${FAILPOINTS:-0} # 1 только на тестовом стенде: файл failpoint-<точка>.png один раз роняет процесс
services:
postgres:
image: postgres:18.6-alpine@sha256:10a5c69a83035ab5239f5620d708520e8b28f5aa1f2b20143e6024532edadfb5
entrypoint: ["/bin/sh", "/etc/lab/entrypoint.sh"]
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql
- ./infra/postgres:/etc/lab:ro
- ./secrets/tls/postgres:/run/identity:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 20
deploy:
resources:
limits: { cpus: "2", memory: 1g }
stop_grace_period: 30s
restart: unless-stopped
security_opt: [no-new-privileges:true]
networks: [backend]
openbao:
image: openbao/openbao:2.5.4@sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115
user: "100:1000"
entrypoint: ["/bin/bao"]
command: ["server", "-config=/etc/lab/server.hcl"]
volumes:
- ./infra/openbao/server.hcl:/etc/lab/server.hcl:ro
- ./secrets/tls/openbao:/run/identity:ro
- bao-data:/openbao/file
- bao-audit:/openbao/logs
networks: [frontend, backend]
ports: ["127.0.0.1:${BAO_PORT:-8200}:8200"]
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
tmpfs: ["/tmp:size=16m,mode=1777"]
mem_limit: 256m
cpus: 0.5
restart: unless-stopped
# Filer, master и volume на loopback в том же контейнере. Наружу слушает только TLS S3.
s3:
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", "-filer", "-s3",
"-master.volumeSizeLimitMB=128", "-volume.minFreeSpace=64MiB", "-s3.ip.bind=0.0.0.0", "-s3.port=9000",
"-s3.config=/run/s3/identities.json", "-s3.cert.file=/run/identity/tls.crt", "-s3.key.file=/run/identity/tls.key",
"-s3.iam=false", "-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.autoCreateBucket=false",
"-s3.concurrentUploadLimitMB=256"]
volumes:
- s3data:/data
- ./secrets/runtime/s3:/run/s3:ro
- ./secrets/tls/s3:/run/identity:ro
- ./secrets/tls/service-ca:/run/tls:ro
healthcheck:
test: ["CMD-SHELL", "SSL_CERT_FILE=/run/tls/ca.crt wget -qO /dev/null https://s3:9000/healthz"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 768m
cpus: 1
restart: unless-stopped
security_opt: [no-new-privileges:true]
networks: [backend]
# Одноразовые сервисы до старта реплик: схема БД (advisory lock) и bucket (повторный запуск ничего не меняет).
migrate:
<<: *app
command: ["node", "src/main.ts", "migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
depends_on:
postgres: { condition: service_healthy }
s3-init:
<<: *app
command: ["node", "src/main.ts", "s3-init"]
environment:
<<: *s3-env
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets:
- { source: s3_admin_access_key_id, target: s3_access_key_id }
- { source: s3_admin_secret_access_key, target: s3_secret_access_key }
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
depends_on:
s3: { condition: service_healthy }
# Брокер: RabbitMQ, quorum-очередь на диске (volume rabbitmq-data), publisher confirms, ручной ack.
# hostname фиксирует имя узла rabbit@rabbitmq: данные в volume привязаны к нему и переживают пересоздание.
rabbitmq:
image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5
hostname: rabbitmq
volumes:
- rabbitmq-data:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
- ./secrets/tls/rabbitmq:/run/identity:ro
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/telemetry-ca:/run/telemetry:ro
secrets: [rabbitmq_definitions]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 30
start_period: 20s
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
networks: [backend]
api:
<<: *app
environment:
<<: [*s3-env, *jobs-env, *otel-env]
OTEL_SERVICE_NAME: messenger-api
BAO_ROLE: api
ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456}
ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2}
ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-2500}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
volumes:
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/api:/run/identity:ro
- ./secrets/pki:/run/pki:ro
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:8081/"]
interval: 2s
timeout: 2s
retries: 3
# Lab6: старт с OpenTelemetry SDK (загрузка instrumentations через ESM-hook) на занятом хосте — до ~25 с.
# Успешная проверка сразу делает контейнер healthy; start_period только прощает неудачи при старте.
start_period: 120s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-256m}" }
stop_grace_period: 15s
restart: unless-stopped # упавшая реплика поднимается сама; после `docker stop` остаётся остановленной
# Worker: отдельный процесс без HTTP, конкурирующие consumers одной очереди. WORKER_CONCURRENCY = prefetch =
# одновременных заданий на реплику; память ограничена контейнером, libvips — один поток, без кэша.
worker:
<<: *app
command: ["node", "--import", "./src/telemetry.ts", "src/main.ts", "worker"]
environment:
<<: [*s3-env, *jobs-env, *otel-env]
OTEL_SERVICE_NAME: messenger-worker
BAO_ROLE: worker
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
volumes:
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/worker:/run/identity:ro
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
rabbitmq: { condition: service_healthy }
deploy:
replicas: ${WORKER_REPLICAS:-2}
resources:
limits: { cpus: "${WORKER_CPUS:-1}", memory: "${WORKER_MEMORY:-768m}" }
stop_grace_period: 15s # SIGTERM: consumer отменяется, текущие задания до 10 с, затем выход
restart: unless-stopped
# Периодическая сверка БД ↔ S3 с grace period (src/reconcile.ts). Свои S3-ключи: объекты + листинг bucket.
reconciler:
<<: *app
command: ["node", "src/main.ts", "reconcile", "--apply", "--every=${RECONCILE_INTERVAL_SECONDS:-300}"]
environment:
<<: *s3-env
BAO_ROLE: reconciler
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
volumes:
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/reconciler:/run/identity:ro
depends_on:
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
deploy:
resources:
limits: { cpus: "0.5", memory: 128m }
restart: unless-stopped
lb:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/tls/ingress:/run/public:ro
- ./secrets/tls/lb-client:/run/client:ro
- ./secrets/tls/lb:/run/identity:ro
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/telemetry-ca:/run/telemetry:ro
ports: ["127.0.0.1:${LB_PORT:-8443}:8443"]
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
depends_on:
api: { condition: service_healthy }
deploy:
resources:
limits: { cpus: "${LB_CPUS:-1}", memory: 128m }
restart: unless-stopped
networks: [frontend, backend]
# HAProxy проверяет CN и CRL перед передачей OTLP по loopback в общем network namespace.
otel-collector:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
volumes:
- ./infra/otel/ingress.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/tls/otel-collector:/run/identity:ro
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/pki:/run/pki:ro
networks: [backend, telemetry]
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 128m
cpus: 0.5
restart: unless-stopped
logging: *logging
collector:
image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849
network_mode: "service:otel-collector"
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/collector:/run/client:ro
environment:
GOMEMLIMIT: 180MiB
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
mem_limit: 256m
cpus: 1
restart: unless-stopped
logging: *logging
prometheus:
image: prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
- --web.config.file=/etc/prometheus/web.yml
- --enable-feature=created-timestamp-zero-ingestion # новая серия счётчика начинается с 0: rate() видит первые ошибки
- --query.lookback-delta=1m # данные остановленной реплики перестают учитываться через минуту, а не через 5
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- ./secrets/tls/prometheus:/run/identity:ro
- ./secrets/tls/telemetry-ca:/run/telemetry:ro
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/prometheus-scrape:/run/client:ro
- prometheus-data:/prometheus
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
logging: *logging
networks: [telemetry]
loki:
image: grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- ./secrets/tls/loki:/run/identity:ro
- ./secrets/tls/telemetry-ca:/run/telemetry:ro
- loki-data:/loki
deploy:
resources:
limits: { cpus: "1", memory: 384m }
stop_grace_period: 30s
restart: unless-stopped
logging: *logging
networks: [telemetry]
tempo:
image: grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- ./secrets/tls/tempo:/run/identity:ro
- ./secrets/tls/telemetry-ca:/run/telemetry:ro
- ./secrets/tls/collector-ca:/run/collector-ca:ro
- tempo-data:/var/tempo
deploy:
resources:
limits: { cpus: "1", memory: 512m }
stop_grace_period: 30s
restart: unless-stopped
logging: *logging
networks: [telemetry]
# Grafana: интерфейс и запросы к хранилищам; datasources и dashboard импортируются из Git (provisioning).
grafana:
image: grafana/grafana:13.2.5@sha256:9f7aa1e1e3246aade5792d35a738049f7f32f89cd51d1956cf426a4e86268a84
entrypoint:
- /bin/sh
- -ec
- |
export DS_CA="$$(cat /run/tls/ca.crt)" DS_CERT="$$(cat /run/client/tls.crt)" DS_KEY="$$(cat /run/client/tls.key)"
exec /run.sh
environment:
GF_SERVER_PROTOCOL: https
GF_SERVER_CERT_FILE: /run/identity/tls.crt
GF_SERVER_CERT_KEY: /run/identity/tls.key
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_SERVER_ROOT_URL: https://localhost:${GRAFANA_PORT:-3000}
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_NEWS_NEWS_FEED_ENABLED: "false"
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ./secrets/tls/grafana:/run/identity:ro
- ./secrets/tls/service-ca:/run/tls:ro
- ./secrets/tls/grafana-query:/run/client:ro
- grafana-data:/var/lib/grafana
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
healthcheck:
test: ["CMD-SHELL", "SSL_CERT_FILE=/run/tls/ca.crt wget -qO /dev/null https://localhost:3000/api/health"]
interval: 5s
timeout: 3s
retries: 30
deploy:
resources:
limits: { cpus: "0.5", memory: 256m }
restart: unless-stopped
logging: *logging
networks: [frontend, telemetry]
networks:
frontend: {}
backend: { internal: true } # api, worker, postgres, rabbitmq и s3-шлюз недоступны с хоста и не имеют выхода наружу
telemetry: { internal: true } # collector → prometheus/loki/tempo ← grafana; приложение сюда не подключено
volumes:
bao-data: {}
bao-audit: {}
pgdata: {}
s3data: {}
rabbitmq-data: {}
prometheus-data: {}
loki-data: {}
tempo-data: {}
grafana-data: {}
secrets:
database_url: { file: ./secrets/runtime/database_url }
postgres_password: { file: ./secrets/runtime/postgres_password }
s3_admin_access_key_id: { file: ./secrets/runtime/s3_admin_access_key_id }
s3_admin_secret_access_key: { file: ./secrets/runtime/s3_admin_secret_access_key }
rabbitmq_definitions: { file: ./secrets/runtime/rabbitmq_definitions.json }
grafana_admin_password: { file: ./secrets/runtime/grafana_admin_password }