732 lines
29 KiB
YAML
732 lines
29 KiB
YAML
name: messenger-lab7
|
||
|
||
# Lab7: безопасность. Порядок первого запуска (REPORT.md, «Запуск из чистого clone»):
|
||
# pki.py init -> init_secrets.py -> up openbao -> bao.py init/unseal/configure -> up --build --wait
|
||
# В репозитории нет ни одного секрета: secrets/ (CA, ключи, пароли) создаёт оператор локально, .gitignore.
|
||
#
|
||
# Сети (internal = без выхода наружу и без публикации портов):
|
||
# ingress — только lb; публикуется 127.0.0.1:8443 (HTTPS) и 127.0.0.1:8404 (статистика, mTLS оператора)
|
||
# frontend — internal: lb <-> api (mTLS, у API только CN=lb)
|
||
# data — internal: приложение <-> PostgreSQL / Redis / S3 / RabbitMQ (все TLS)
|
||
# secrets — internal: сервисы <-> OpenBao (TLS + cert auth)
|
||
# telemetry — internal: приложение -> otel-gate (mTLS) -> Collector -> Prometheus/Loki/Tempo (mTLS) <- Grafana
|
||
# ops — только для публикации админ-интерфейсов на 127.0.0.1 (OpenBao, Prometheus, Loki, Tempo, Grafana)
|
||
|
||
x-app-build: &app-build
|
||
context: .
|
||
args:
|
||
APP_VERSION: ${APP_VERSION:-dev}
|
||
|
||
# Общие меры для контейнеров: только чтение корня, без capabilities, без повышения прав, не root
|
||
# (USER 10001 в Dockerfile приложения; у остальных — user образа). Пишут только в tmpfs /tmp и свои volumes.
|
||
# Число процессов ограничено в deploy.resources.limits.pids каждого сервиса.
|
||
x-hardening: &hardening
|
||
read_only: true
|
||
cap_drop: [ALL]
|
||
security_opt: ["no-new-privileges:true"]
|
||
|
||
# TLS-файлы процессов приложения (смонтированы каталогами: ротация leaf видна без пересоздания контейнера)
|
||
# и адрес OpenBao. BAO_ROLE = CN сертификата сервиса = роль cert auth = policy.
|
||
x-tls-env: &tls-env
|
||
TLS_CA_FILE: /run/tls-ca/ca.crt
|
||
TLS_CRL_FILE: /run/tls-ca/crl.pem
|
||
TLS_CERT_FILE: /run/tls/tls.crt
|
||
TLS_KEY_FILE: /run/tls/tls.key
|
||
BAO_ADDR: https://openbao:8200
|
||
SECRETS_REFRESH_SECONDS: ${SECRETS_REFRESH_SECONDS:-30}
|
||
|
||
x-db-env: &db-env
|
||
DB_HOST: postgres
|
||
DB_PORT: "5432"
|
||
DB_NAME: messenger
|
||
DB_SSLMODE: verify-full
|
||
|
||
x-pipeline-env: &pipeline-env
|
||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
||
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
|
||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||
# Учебный стенд: точки внедрения сбоев (срабатывают только по файлу в volume faults worker/dispatcher).
|
||
FAULT_INJECTION: ${FAULT_INJECTION:-1}
|
||
FAULTS_DIR: /faults
|
||
|
||
# lab6 -> lab7: OTLP/HTTP по mTLS к otel-gate (имя otel-collector), сертификат сервиса = его identity.
|
||
x-otel-env: &otel-env
|
||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls-ca/ca.crt
|
||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/tls.crt
|
||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/tls.key
|
||
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
|
||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||
OTEL_METRIC_EXPORT_INTERVAL: "10000"
|
||
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
|
||
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
|
||
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"
|
||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||
|
||
x-otel-depends: &otel-depends
|
||
condition: service_started
|
||
required: false
|
||
|
||
x-alive-check: &alive-check
|
||
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 30s
|
||
start_interval: 1s
|
||
|
||
x-bao-ready: &bao-ready
|
||
condition: service_healthy
|
||
|
||
services:
|
||
# ---------------- секреты ----------------
|
||
# OpenBao 2.6.2, НЕ dev-mode: file storage в volume, TLS-listener, после старта sealed до unseal оператором.
|
||
# healthy = unsealed (bao status: 0 — unsealed, 2 — sealed). 127.0.0.1:8200 — только для оператора.
|
||
openbao:
|
||
image: openbao/openbao:2.6.2
|
||
command: ["server"] # entrypoint образа: bao server -config=/openbao/config
|
||
user: "100" # пользователь openbao образа, не root
|
||
environment:
|
||
SKIP_SETCAP: "true"
|
||
SKIP_CHOWN: "true"
|
||
BAO_ADDR: https://127.0.0.1:8200
|
||
BAO_CACERT: /run/tls-ca/ca.crt
|
||
volumes:
|
||
- ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro
|
||
- ./secrets/pki/openbao:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- openbaodata:/openbao/file
|
||
ports:
|
||
- "127.0.0.1:8200:8200"
|
||
networks: [secrets, ops]
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||
healthcheck:
|
||
test: ["CMD", "bao", "status", "-address=https://127.0.0.1:8200", "-ca-cert=/run/tls-ca/ca.crt"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 5s
|
||
restart: unless-stopped
|
||
stop_grace_period: 15s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||
|
||
# ---------------- хранилища ----------------
|
||
postgres:
|
||
image: postgres:17.11-alpine
|
||
user: "70:70" # postgres образа alpine; entrypoint не делает chown (не нужен CAP_CHOWN)
|
||
environment:
|
||
POSTGRES_USER: messenger
|
||
POSTGRES_DB: messenger
|
||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||
# Ключ TLS копируется в tmpfs с правами 0600 (PostgreSQL отказывается от ключа, доступного группе/всем,
|
||
# а права bind mount на Windows не управляются). Только hostssl + SCRAM (infra/postgres/pg_hba.conf).
|
||
entrypoint: ["sh", "-c", "install -d -m 0700 /tmp/pgtls && install -m 0600 /run/tls/tls.key /tmp/pgtls/tls.key && install -m 0644 /run/tls/tls.crt /tmp/pgtls/tls.crt && exec docker-entrypoint.sh \"$$@\"", "--"]
|
||
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on",
|
||
"-c", "ssl=on", "-c", "ssl_cert_file=/tmp/pgtls/tls.crt", "-c", "ssl_key_file=/tmp/pgtls/tls.key",
|
||
"-c", "ssl_min_protocol_version=TLSv1.2", "-c", "password_encryption=scram-sha-256",
|
||
"-c", "hba_file=/etc/postgresql/pg_hba.conf", "-c", "log_connections=on"]
|
||
secrets: [postgres_password]
|
||
volumes:
|
||
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||
- ./secrets/pki/postgres:/run/tls:ro
|
||
- pgdata:/var/lib/postgresql/data
|
||
networks: [data]
|
||
cap_drop: [ALL]
|
||
security_opt: ["no-new-privileges:true"]
|
||
read_only: true
|
||
tmpfs: ["/tmp:size=16m,mode=1777", "/var/run/postgresql:size=1m,uid=70,gid=70,mode=0775"]
|
||
shm_size: 128m
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] # unix-socket внутри контейнера
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 12
|
||
start_period: 10s
|
||
restart: unless-stopped
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||
|
||
redis:
|
||
image: redis:7.4.11-alpine
|
||
user: "999:1000" # redis образа; без root и без chown
|
||
# Стартовый ACL копируется в volume один раз; дальше ACL живёт в volume (ACL SAVE после ротации).
|
||
command: ["sh", "-c", "[ -f /data/users.acl ] || cp /run/redis-acl/users.acl /data/users.acl; exec redis-server /usr/local/etc/redis/redis.conf"]
|
||
volumes:
|
||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||
- ./secrets/redis:/run/redis-acl:ro
|
||
- ./secrets/pki/redis:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- redisdata:/data
|
||
networks: [data]
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||
healthcheck:
|
||
# Жив и отвечает по TLS; без AUTH сервер отвечает NOAUTH — это и есть «работает и закрыт».
|
||
test: ["CMD-SHELL", "redis-cli --tls --cacert /run/tls-ca/ca.crt --cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis ping 2>&1 | grep -qE 'NOAUTH|PONG'"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 12
|
||
restart: unless-stopped
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||
|
||
# SeaweedFS 4.52: master/volume/filer слушают только 127.0.0.1 (-ip.bind), наружу — только S3 HTTPS :8333
|
||
# (-s3.ip.bind=0.0.0.0) и gRPC шлюза :18333 под mTLS CN=s3 (security.toml). Iceberg/Lance/IAM API выключены.
|
||
s3:
|
||
image: chrislusf/seaweedfs:4.52
|
||
entrypoint: ["/usr/bin/weed"]
|
||
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
|
||
"-master.volumeSizeLimitMB=1024", "-volume.max=0", "-master.telemetry=false",
|
||
"-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0",
|
||
"-s3.cert.file=/run/tls/tls.crt", "-s3.key.file=/run/tls/tls.key",
|
||
"-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.iam=false",
|
||
"-s3.concurrentUploadLimitMB=128",
|
||
"-s3.config=/run/secrets/s3_config"]
|
||
# Docker-прогон: ~240 MiB в простое, 20 параллельных загрузок по ~10 MiB доводили процесс до лимита 512 MiB
|
||
# и SIGKILL (cgroup OOM) — API получал 503. Go GC теперь ориентируется на лимит, а сверх 128 MiB тел
|
||
# загрузок одновременно S3-шлюз ставит запросы в очередь вместо роста памяти.
|
||
environment:
|
||
GOMEMLIMIT: 400MiB
|
||
secrets: [s3_config]
|
||
volumes:
|
||
- ./infra/seaweedfs/security.toml:/etc/seaweedfs/security.toml:ro
|
||
- ./secrets/pki/s3:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- s3data:/data
|
||
networks: [data]
|
||
# Пользователь seaweed образа 4.52 (uid/gid 1000, владелец /data). Entrypoint образа делает chown + su-exec
|
||
# от root; мы его не используем (entrypoint = weed), поэтому сразу не root и без capabilities.
|
||
user: "1000:1000"
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||
healthcheck:
|
||
# Готовность к ЗАПИСИ: после рестарта мастер ~15 c отвечает "master is warming up" на
|
||
# /dir/assign, хотя /cluster/healthz уже 200. Проверяем, что мастер выдаёт fid.
|
||
test: ["CMD-SHELL", "wget -q -O - 'http://127.0.0.1:9333/dir/assign?count=1' | grep -q '\"fid\"'"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 24
|
||
start_period: 15s
|
||
start_interval: 1s
|
||
restart: unless-stopped
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||
|
||
rabbitmq:
|
||
image: rabbitmq:4.1.4-alpine
|
||
hostname: rabbitmq
|
||
user: rabbitmq
|
||
environment:
|
||
RABBITMQ_NODENAME: rabbit@localhost # distribution только на loopback (infra/rabbitmq.conf)
|
||
ERL_EPMD_ADDRESS: 127.0.0.1
|
||
volumes:
|
||
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
|
||
- ./infra/rabbitmq-enabled-plugins:/etc/rabbitmq/enabled_plugins:ro
|
||
- ./secrets/pki/rabbitmq:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- rabbitmqdata:/var/lib/rabbitmq
|
||
secrets: [rabbitmq_definitions]
|
||
networks: [data]
|
||
cap_drop: [ALL]
|
||
security_opt: ["no-new-privileges:true"]
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
|
||
interval: 5s
|
||
timeout: 10s
|
||
retries: 24
|
||
start_period: 20s
|
||
start_interval: 2s
|
||
restart: unless-stopped
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 512}
|
||
|
||
# ---------------- bootstrap ----------------
|
||
migrate:
|
||
build: *app-build
|
||
command: ["python", "migrate.py"]
|
||
environment:
|
||
<<: [*tls-env, *db-env]
|
||
BAO_ROLE: migrate
|
||
DB_USER: messenger
|
||
DB_PASSWORD_KEY: db_owner_password
|
||
volumes:
|
||
- ./secrets/pki/migrate:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
networks: [data, secrets]
|
||
depends_on:
|
||
postgres: {condition: service_healthy}
|
||
openbao: *bao-ready
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||
healthcheck:
|
||
disable: true
|
||
restart: "no"
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||
|
||
s3-init:
|
||
build: *app-build
|
||
command: ["python", "s3_init.py"]
|
||
environment:
|
||
<<: *tls-env
|
||
BAO_ROLE: s3-init
|
||
S3_ENDPOINT: https://s3:8333
|
||
S3_REGION: ${S3_REGION:-us-east-1}
|
||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||
volumes:
|
||
- ./secrets/pki/s3-init:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
networks: [data, secrets]
|
||
depends_on:
|
||
s3: {condition: service_healthy}
|
||
openbao: *bao-ready
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||
healthcheck:
|
||
disable: true
|
||
restart: "no"
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||
|
||
# ---------------- приложение ----------------
|
||
api:
|
||
build: *app-build
|
||
env_file: .env
|
||
environment:
|
||
<<: [*tls-env, *db-env, *otel-env]
|
||
APP_HOST: 0.0.0.0
|
||
APP_PORT: "8443"
|
||
API_TLS: "1"
|
||
API_ALLOWED_CLIENTS: lb # identity, которой разрешено подключаться к API (mTLS)
|
||
BAO_ROLE: api
|
||
DB_USER: messenger_api
|
||
REDIS_HOST: redis
|
||
REDIS_USER: api
|
||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||
S3_ENDPOINT: https://s3:8333
|
||
BROKER_URL: "" # у API нет доступа к брокеру: задание уходит через outbox
|
||
UPLOAD_TMP_DIR: /tmp/uploads
|
||
# Буферы Argon2id (32 MiB) — всегда через mmap и сразу назад ОС. Без этого glibc оставляет освобождённые
|
||
# буферы в per-thread arenas пула потоков Starlette: замер (login_dos.py, 200 проверок) — RSS +548 MiB
|
||
# против +37 MiB с этой настройкой; при лимите контейнера 512 MiB это разница между OOM и запасом.
|
||
MALLOC_MMAP_THRESHOLD_: "1048576"
|
||
OTEL_SERVICE_NAME: messenger-api
|
||
volumes:
|
||
- ./secrets/pki/api:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
networks: [frontend, data, secrets, telemetry]
|
||
depends_on:
|
||
migrate: {condition: service_completed_successfully}
|
||
s3-init: {condition: service_completed_successfully}
|
||
redis: {condition: service_healthy}
|
||
openbao: *bao-ready
|
||
otel-gate: *otel-depends
|
||
<<: *hardening
|
||
# Временные файлы текущих загрузок (до 10 MiB каждый) — tmpfs, учитывается в лимите памяти контейнера.
|
||
tmpfs: ["/tmp:size=160m,mode=1777"]
|
||
healthcheck:
|
||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2)"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 30s
|
||
start_interval: 1s
|
||
restart: unless-stopped
|
||
stop_grace_period: 15s
|
||
deploy:
|
||
replicas: ${API_REPLICAS:-2}
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||
|
||
janitor:
|
||
build: *app-build
|
||
command: ["python", "janitor.py"]
|
||
env_file: .env
|
||
environment:
|
||
<<: [*tls-env, *db-env, *otel-env]
|
||
BAO_ROLE: janitor
|
||
DB_USER: messenger_janitor
|
||
S3_ENDPOINT: https://s3:8333
|
||
BROKER_URL: ""
|
||
OTEL_SERVICE_NAME: messenger-janitor
|
||
volumes:
|
||
- ./secrets/pki/janitor:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
networks: [data, secrets, telemetry]
|
||
depends_on:
|
||
migrate: {condition: service_completed_successfully}
|
||
s3-init: {condition: service_completed_successfully}
|
||
openbao: *bao-ready
|
||
otel-gate: *otel-depends
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||
healthcheck: *alive-check
|
||
restart: unless-stopped
|
||
stop_grace_period: 10s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.25", memory: 192M, pids: 256}
|
||
|
||
dispatcher:
|
||
build: *app-build
|
||
command: ["python", "dispatcher.py"]
|
||
environment:
|
||
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
|
||
BAO_ROLE: dispatcher
|
||
DB_USER: messenger_dispatcher
|
||
BROKER_URL: amqps://rabbitmq:5671/messenger # не секрет: вход по сертификату CN=dispatcher
|
||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||
volumes:
|
||
- ./secrets/pki/dispatcher:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- faults:/faults
|
||
networks: [data, secrets, telemetry]
|
||
depends_on:
|
||
migrate: {condition: service_completed_successfully}
|
||
rabbitmq: {condition: service_healthy}
|
||
openbao: *bao-ready
|
||
otel-gate: *otel-depends
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||
healthcheck: *alive-check
|
||
restart: unless-stopped
|
||
stop_grace_period: 10s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||
|
||
# Worker: обработка изображения в дочернем процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB,
|
||
# RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS; контейнер ограничен по CPU/RAM/PID.
|
||
worker:
|
||
build: *app-build
|
||
command: ["python", "worker.py"]
|
||
environment:
|
||
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
|
||
BAO_ROLE: worker
|
||
DB_USER: messenger_worker
|
||
BROKER_URL: amqps://rabbitmq:5671/messenger # вход по сертификату CN=worker
|
||
S3_ENDPOINT: https://s3:8333
|
||
S3_REGION: ${S3_REGION:-us-east-1}
|
||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
|
||
WORKER_TMP_DIR: /tmp/work
|
||
OTEL_SERVICE_NAME: messenger-worker
|
||
volumes:
|
||
- ./secrets/pki/worker:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- faults:/faults
|
||
networks: [data, secrets, telemetry]
|
||
depends_on:
|
||
migrate: {condition: service_completed_successfully}
|
||
s3-init: {condition: service_completed_successfully}
|
||
rabbitmq: {condition: service_healthy}
|
||
openbao: *bao-ready
|
||
otel-gate: *otel-depends
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=256m,mode=1777"]
|
||
healthcheck: *alive-check
|
||
restart: unless-stopped
|
||
stop_grace_period: 70s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "2.00", memory: 1536M, pids: 128}
|
||
|
||
lb:
|
||
image: haproxy:3.2.25-alpine
|
||
ports:
|
||
- "127.0.0.1:8443:8443"
|
||
- "127.0.0.1:8404:8404"
|
||
volumes:
|
||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||
- ./secrets/pki/ingress:/run/tls-ingress:ro
|
||
- ./secrets/pki/lb:/run/tls-lb:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
networks: [ingress, frontend]
|
||
depends_on:
|
||
api: {condition: service_healthy}
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
|
||
healthcheck:
|
||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/lb-health"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 10s
|
||
start_interval: 1s
|
||
restart: unless-stopped
|
||
stop_grace_period: 15s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||
|
||
# ---------------- наблюдаемость ----------------
|
||
# otel-gate владеет network namespace; Collector живёт в нём же и слушает только 127.0.0.1.
|
||
# Имя otel-collector в сети telemetry указывает на gate (alias): приложение ходит на https://otel-collector:4318.
|
||
otel-gate:
|
||
image: haproxy:3.2.25-alpine
|
||
volumes:
|
||
- ./infra/otel/gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||
- ./secrets/pki/otel-collector:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
networks:
|
||
telemetry:
|
||
aliases: [otel-collector]
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
|
||
healthcheck:
|
||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/gate-health"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 3
|
||
start_period: 5s
|
||
restart: unless-stopped
|
||
stop_grace_period: 10s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.25", memory: 64M, pids: 256}
|
||
|
||
otel-collector:
|
||
image: otel/opentelemetry-collector-contrib:0.164.0
|
||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||
network_mode: "service:otel-gate"
|
||
volumes:
|
||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||
- ./secrets/pki/otel-collector:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
depends_on:
|
||
otel-gate: {condition: service_started}
|
||
<<: *hardening
|
||
restart: unless-stopped
|
||
stop_grace_period: 10s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 384M, pids: 256}
|
||
|
||
# Одноразовый chown volumes Loki/Tempo для uid 10001 (их процессы — не root). Исключение: root + CAP_CHOWN,
|
||
# без сети, завершается сразу. Идемпотентно: при повторном up каталоги уже 10001 и внутри — 0700-каталоги
|
||
# Loki/Tempo, в которые root без CAP_DAC_READ_SEARCH не войдёт, поэтому chown только для ещё чужого volume.
|
||
telemetry-init:
|
||
image: busybox:1.37.0
|
||
user: root
|
||
command: ["sh", "-c", "for d in /loki /tempo; do [ \"$$(stat -c %u $$d)\" = 10001 ] || chown -R 10001:10001 $$d || exit 1; done"]
|
||
volumes:
|
||
- lokidata:/loki
|
||
- tempodata:/tempo
|
||
network_mode: none
|
||
cap_drop: [ALL]
|
||
cap_add: [CHOWN]
|
||
security_opt: ["no-new-privileges:true"]
|
||
read_only: true
|
||
restart: "no"
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.25", memory: 32M, pids: 256}
|
||
|
||
prometheus:
|
||
image: prom/prometheus:v3.15.1
|
||
command:
|
||
- --config.file=/etc/prometheus/prometheus.yml
|
||
- --web.config.file=/etc/prometheus/web.yml
|
||
- --storage.tsdb.path=/prometheus
|
||
- --storage.tsdb.retention.time=7d
|
||
- --storage.tsdb.retention.size=1GB
|
||
- --web.enable-otlp-receiver
|
||
volumes:
|
||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||
- ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro
|
||
- ./infra/prometheus/promtool-http.yml:/etc/prometheus/promtool-http.yml:ro
|
||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||
- ./infra/prometheus/tests:/etc/prometheus/tests:ro
|
||
- ./secrets/pki/prometheus:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- promdata:/prometheus
|
||
ports:
|
||
- "127.0.0.1:9090:9090"
|
||
networks: [telemetry, ops]
|
||
<<: *hardening
|
||
# promtool test rules (unit-тесты алертов, check_configs.py) создаёт временную TSDB в /tmp
|
||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||
healthcheck:
|
||
test: ["CMD", "promtool", "check", "ready", "--url=https://localhost:9090", "--http.config.file=/etc/prometheus/promtool-http.yml"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 24
|
||
start_period: 10s
|
||
restart: unless-stopped
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||
|
||
loki:
|
||
image: grafana/loki:3.7.9
|
||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||
volumes:
|
||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||
- ./secrets/pki/loki:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- lokidata:/loki
|
||
ports:
|
||
- "127.0.0.1:3100:3100"
|
||
networks: [telemetry, ops]
|
||
depends_on:
|
||
telemetry-init: {condition: service_completed_successfully}
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||
restart: unless-stopped
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||
|
||
tempo:
|
||
image: grafana/tempo:3.1.1
|
||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||
volumes:
|
||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||
- ./secrets/pki/tempo:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- tempodata:/var/tempo
|
||
ports:
|
||
- "127.0.0.1:3200:3200"
|
||
networks: [telemetry, ops]
|
||
depends_on:
|
||
telemetry-init: {condition: service_completed_successfully}
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||
restart: unless-stopped
|
||
stop_grace_period: 30s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||
|
||
grafana:
|
||
image: grafana/grafana:12.4.14
|
||
environment:
|
||
GF_SERVER_PROTOCOL: https
|
||
GF_SERVER_CERT_FILE: /run/tls/tls.crt
|
||
GF_SERVER_CERT_KEY: /run/tls/tls.key
|
||
GF_SERVER_DOMAIN: localhost
|
||
GF_SERVER_ROOT_URL: https://localhost:3000/
|
||
GF_SECURITY_ADMIN_USER: admin
|
||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||
GF_SECURITY_COOKIE_SECURE: "true"
|
||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
|
||
secrets: [grafana_admin_password]
|
||
volumes:
|
||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
|
||
- ./secrets/pki/grafana:/run/tls:ro
|
||
- ./secrets/pki/public:/run/tls-ca:ro
|
||
- grafanadata:/var/lib/grafana
|
||
ports:
|
||
- "127.0.0.1:3000:3000"
|
||
networks: [telemetry, ops]
|
||
depends_on:
|
||
prometheus: {condition: service_healthy}
|
||
loki: {condition: service_started}
|
||
tempo: {condition: service_started}
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "curl -fsS -m 2 -o /dev/null --cacert /run/tls-ca/ca.crt https://localhost:3000/api/health || wget -q -T 2 -O /dev/null https://localhost:3000/api/health"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 24
|
||
start_period: 20s
|
||
restart: unless-stopped
|
||
stop_grace_period: 15s
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 384M, pids: 256}
|
||
|
||
# ---------------- только для проверок (профиль tools, сам не стартует) ----------------
|
||
# Контейнер-зонд во всех внутренних сетях: отрицательные/положительные проверки trust matrix изнутри
|
||
# (student-tests/tls_probe.py). Получает ТОЛЬКО тестовые сертификаты и сертификаты клиентов, которыми
|
||
# пробует войти «не туда»; ключа CA и секретов OpenBao у него нет.
|
||
probe:
|
||
build: *app-build
|
||
profiles: [tools]
|
||
command: ["python", "/probe/tls_probe.py", "matrix"]
|
||
environment:
|
||
PKI: /pki
|
||
PYTHONPATH: /app
|
||
volumes:
|
||
- ./student-tests:/probe:ro
|
||
- ./secrets/pki/public:/pki/public:ro
|
||
- ./secrets/pki/test:/pki/test:ro
|
||
- ./secrets/pki/lb:/pki/lb:ro
|
||
- ./secrets/pki/api:/pki/api:ro
|
||
- ./secrets/pki/worker:/pki/worker:ro
|
||
- ./secrets/pki/dispatcher:/pki/dispatcher:ro
|
||
- ./secrets/pki/janitor:/pki/janitor:ro
|
||
- ./secrets/pki/grafana:/pki/grafana:ro
|
||
- ./secrets/pki/otel-collector:/pki/otel-collector:ro
|
||
- ./secrets/pki/operator:/pki/operator:ro
|
||
networks: [frontend, data, secrets, telemetry]
|
||
<<: *hardening
|
||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||
restart: "no"
|
||
deploy:
|
||
resources:
|
||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||
|
||
networks:
|
||
ingress: {}
|
||
frontend:
|
||
internal: true
|
||
data:
|
||
internal: true
|
||
secrets:
|
||
internal: true
|
||
telemetry:
|
||
internal: true
|
||
# Только для публикации админ-интерфейсов на 127.0.0.1 (internal-сеть не может публиковать порты).
|
||
ops: {}
|
||
|
||
volumes:
|
||
openbaodata: {}
|
||
pgdata: {}
|
||
redisdata: {}
|
||
s3data: {}
|
||
rabbitmqdata: {}
|
||
faults: {}
|
||
promdata: {}
|
||
lokidata: {}
|
||
tempodata: {}
|
||
grafanadata: {}
|
||
|
||
secrets:
|
||
postgres_password: {file: ./secrets/postgres_password}
|
||
s3_config: {file: ./secrets/s3_config.json}
|
||
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
|
||
grafana_admin_password: {file: ./secrets/grafana_admin_password}
|