forked from mtusi2026/mtusi2026lab7
лр7: Docker-прогон, evidence, исправления
This commit is contained in:
+731
@@ -0,0 +1,731 @@
|
||||
name: messenger-lab7
|
||||
|
||||
# Lab7: безопасность. Порядок первого запуска (REPORT.md, «Запуск из чистого clone»):
|
||||
# pki.py init -> init_secrets.py -> up openbao -> bao.py init/unseal/configure -> up --build --wait
|
||||
# В репозитории нет ни одного секрета: secrets/ (CA, ключи, пароли) создаёт оператор локально, .gitignore.
|
||||
#
|
||||
# Сети (internal = без выхода наружу и без публикации портов):
|
||||
# ingress — только lb; публикуется 127.0.0.1:8443 (HTTPS) и 127.0.0.1:8404 (статистика, mTLS оператора)
|
||||
# frontend — internal: lb <-> api (mTLS, у API только CN=lb)
|
||||
# data — internal: приложение <-> PostgreSQL / Redis / S3 / RabbitMQ (все TLS)
|
||||
# secrets — internal: сервисы <-> OpenBao (TLS + cert auth)
|
||||
# telemetry — internal: приложение -> otel-gate (mTLS) -> Collector -> Prometheus/Loki/Tempo (mTLS) <- Grafana
|
||||
# ops — только для публикации админ-интерфейсов на 127.0.0.1 (OpenBao, Prometheus, Loki, Tempo, Grafana)
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
# Общие меры для контейнеров: только чтение корня, без capabilities, без повышения прав, не root
|
||||
# (USER 10001 в Dockerfile приложения; у остальных — user образа). Пишут только в tmpfs /tmp и свои volumes.
|
||||
# Число процессов ограничено в deploy.resources.limits.pids каждого сервиса.
|
||||
x-hardening: &hardening
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
|
||||
# TLS-файлы процессов приложения (смонтированы каталогами: ротация leaf видна без пересоздания контейнера)
|
||||
# и адрес OpenBao. BAO_ROLE = CN сертификата сервиса = роль cert auth = policy.
|
||||
x-tls-env: &tls-env
|
||||
TLS_CA_FILE: /run/tls-ca/ca.crt
|
||||
TLS_CRL_FILE: /run/tls-ca/crl.pem
|
||||
TLS_CERT_FILE: /run/tls/tls.crt
|
||||
TLS_KEY_FILE: /run/tls/tls.key
|
||||
BAO_ADDR: https://openbao:8200
|
||||
SECRETS_REFRESH_SECONDS: ${SECRETS_REFRESH_SECONDS:-30}
|
||||
|
||||
x-db-env: &db-env
|
||||
DB_HOST: postgres
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: messenger
|
||||
DB_SSLMODE: verify-full
|
||||
|
||||
x-pipeline-env: &pipeline-env
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||||
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
||||
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
# Учебный стенд: точки внедрения сбоев (срабатывают только по файлу в volume faults worker/dispatcher).
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-1}
|
||||
FAULTS_DIR: /faults
|
||||
|
||||
# lab6 -> lab7: OTLP/HTTP по mTLS к otel-gate (имя otel-collector), сертификат сервиса = его identity.
|
||||
x-otel-env: &otel-env
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls-ca/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/tls.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/tls.key
|
||||
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||||
OTEL_METRIC_EXPORT_INTERVAL: "10000"
|
||||
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
|
||||
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
|
||||
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
|
||||
x-otel-depends: &otel-depends
|
||||
condition: service_started
|
||||
required: false
|
||||
|
||||
x-alive-check: &alive-check
|
||||
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
|
||||
x-bao-ready: &bao-ready
|
||||
condition: service_healthy
|
||||
|
||||
services:
|
||||
# ---------------- секреты ----------------
|
||||
# OpenBao 2.6.2, НЕ dev-mode: file storage в volume, TLS-listener, после старта sealed до unseal оператором.
|
||||
# healthy = unsealed (bao status: 0 — unsealed, 2 — sealed). 127.0.0.1:8200 — только для оператора.
|
||||
openbao:
|
||||
image: openbao/openbao:2.6.2
|
||||
command: ["server"] # entrypoint образа: bao server -config=/openbao/config
|
||||
user: "100" # пользователь openbao образа, не root
|
||||
environment:
|
||||
SKIP_SETCAP: "true"
|
||||
SKIP_CHOWN: "true"
|
||||
BAO_ADDR: https://127.0.0.1:8200
|
||||
BAO_CACERT: /run/tls-ca/ca.crt
|
||||
volumes:
|
||||
- ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro
|
||||
- ./secrets/pki/openbao:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- openbaodata:/openbao/file
|
||||
ports:
|
||||
- "127.0.0.1:8200:8200"
|
||||
networks: [secrets, ops]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD", "bao", "status", "-address=https://127.0.0.1:8200", "-ca-cert=/run/tls-ca/ca.crt"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||||
|
||||
# ---------------- хранилища ----------------
|
||||
postgres:
|
||||
image: postgres:17.11-alpine
|
||||
user: "70:70" # postgres образа alpine; entrypoint не делает chown (не нужен CAP_CHOWN)
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
# Ключ TLS копируется в tmpfs с правами 0600 (PostgreSQL отказывается от ключа, доступного группе/всем,
|
||||
# а права bind mount на Windows не управляются). Только hostssl + SCRAM (infra/postgres/pg_hba.conf).
|
||||
entrypoint: ["sh", "-c", "install -d -m 0700 /tmp/pgtls && install -m 0600 /run/tls/tls.key /tmp/pgtls/tls.key && install -m 0644 /run/tls/tls.crt /tmp/pgtls/tls.crt && exec docker-entrypoint.sh \"$$@\"", "--"]
|
||||
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on",
|
||||
"-c", "ssl=on", "-c", "ssl_cert_file=/tmp/pgtls/tls.crt", "-c", "ssl_key_file=/tmp/pgtls/tls.key",
|
||||
"-c", "ssl_min_protocol_version=TLSv1.2", "-c", "password_encryption=scram-sha-256",
|
||||
"-c", "hba_file=/etc/postgresql/pg_hba.conf", "-c", "log_connections=on"]
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||||
- ./secrets/pki/postgres:/run/tls:ro
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [data]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:size=16m,mode=1777", "/var/run/postgresql:size=1m,uid=70,gid=70,mode=0775"]
|
||||
shm_size: 128m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] # unix-socket внутри контейнера
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
redis:
|
||||
image: redis:7.4.11-alpine
|
||||
user: "999:1000" # redis образа; без root и без chown
|
||||
# Стартовый ACL копируется в volume один раз; дальше ACL живёт в volume (ACL SAVE после ротации).
|
||||
command: ["sh", "-c", "[ -f /data/users.acl ] || cp /run/redis-acl/users.acl /data/users.acl; exec redis-server /usr/local/etc/redis/redis.conf"]
|
||||
volumes:
|
||||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
- ./secrets/redis:/run/redis-acl:ro
|
||||
- ./secrets/pki/redis:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- redisdata:/data
|
||||
networks: [data]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||||
healthcheck:
|
||||
# Жив и отвечает по TLS; без AUTH сервер отвечает NOAUTH — это и есть «работает и закрыт».
|
||||
test: ["CMD-SHELL", "redis-cli --tls --cacert /run/tls-ca/ca.crt --cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis ping 2>&1 | grep -qE 'NOAUTH|PONG'"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||||
|
||||
# SeaweedFS 4.52: master/volume/filer слушают только 127.0.0.1 (-ip.bind), наружу — только S3 HTTPS :8333
|
||||
# (-s3.ip.bind=0.0.0.0) и gRPC шлюза :18333 под mTLS CN=s3 (security.toml). Iceberg/Lance/IAM API выключены.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.52
|
||||
entrypoint: ["/usr/bin/weed"]
|
||||
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
|
||||
"-master.volumeSizeLimitMB=1024", "-volume.max=0", "-master.telemetry=false",
|
||||
"-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0",
|
||||
"-s3.cert.file=/run/tls/tls.crt", "-s3.key.file=/run/tls/tls.key",
|
||||
"-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.iam=false",
|
||||
"-s3.concurrentUploadLimitMB=128",
|
||||
"-s3.config=/run/secrets/s3_config"]
|
||||
# Docker-прогон: ~240 MiB в простое, 20 параллельных загрузок по ~10 MiB доводили процесс до лимита 512 MiB
|
||||
# и SIGKILL (cgroup OOM) — API получал 503. Go GC теперь ориентируется на лимит, а сверх 128 MiB тел
|
||||
# загрузок одновременно S3-шлюз ставит запросы в очередь вместо роста памяти.
|
||||
environment:
|
||||
GOMEMLIMIT: 400MiB
|
||||
secrets: [s3_config]
|
||||
volumes:
|
||||
- ./infra/seaweedfs/security.toml:/etc/seaweedfs/security.toml:ro
|
||||
- ./secrets/pki/s3:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- s3data:/data
|
||||
networks: [data]
|
||||
# Пользователь seaweed образа 4.52 (uid/gid 1000, владелец /data). Entrypoint образа делает chown + su-exec
|
||||
# от root; мы его не используем (entrypoint = weed), поэтому сразу не root и без capabilities.
|
||||
user: "1000:1000"
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
healthcheck:
|
||||
# Готовность к ЗАПИСИ: после рестарта мастер ~15 c отвечает "master is warming up" на
|
||||
# /dir/assign, хотя /cluster/healthz уже 200. Проверяем, что мастер выдаёт fid.
|
||||
test: ["CMD-SHELL", "wget -q -O - 'http://127.0.0.1:9333/dir/assign?count=1' | grep -q '\"fid\"'"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 15s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
hostname: rabbitmq
|
||||
user: rabbitmq
|
||||
environment:
|
||||
RABBITMQ_NODENAME: rabbit@localhost # distribution только на loopback (infra/rabbitmq.conf)
|
||||
ERL_EPMD_ADDRESS: 127.0.0.1
|
||||
volumes:
|
||||
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
|
||||
- ./infra/rabbitmq-enabled-plugins:/etc/rabbitmq/enabled_plugins:ro
|
||||
- ./secrets/pki/rabbitmq:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
secrets: [rabbitmq_definitions]
|
||||
networks: [data]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 24
|
||||
start_period: 20s
|
||||
start_interval: 2s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 512}
|
||||
|
||||
# ---------------- bootstrap ----------------
|
||||
migrate:
|
||||
build: *app-build
|
||||
command: ["python", "migrate.py"]
|
||||
environment:
|
||||
<<: [*tls-env, *db-env]
|
||||
BAO_ROLE: migrate
|
||||
DB_USER: messenger
|
||||
DB_PASSWORD_KEY: db_owner_password
|
||||
volumes:
|
||||
- ./secrets/pki/migrate:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [data, secrets]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||||
|
||||
s3-init:
|
||||
build: *app-build
|
||||
command: ["python", "s3_init.py"]
|
||||
environment:
|
||||
<<: *tls-env
|
||||
BAO_ROLE: s3-init
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
volumes:
|
||||
- ./secrets/pki/s3-init:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [data, secrets]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||||
|
||||
# ---------------- приложение ----------------
|
||||
api:
|
||||
build: *app-build
|
||||
env_file: .env
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *otel-env]
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8443"
|
||||
API_TLS: "1"
|
||||
API_ALLOWED_CLIENTS: lb # identity, которой разрешено подключаться к API (mTLS)
|
||||
BAO_ROLE: api
|
||||
DB_USER: messenger_api
|
||||
REDIS_HOST: redis
|
||||
REDIS_USER: api
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
BROKER_URL: "" # у API нет доступа к брокеру: задание уходит через outbox
|
||||
UPLOAD_TMP_DIR: /tmp/uploads
|
||||
# Буферы Argon2id (32 MiB) — всегда через mmap и сразу назад ОС. Без этого glibc оставляет освобождённые
|
||||
# буферы в per-thread arenas пула потоков Starlette: замер (login_dos.py, 200 проверок) — RSS +548 MiB
|
||||
# против +37 MiB с этой настройкой; при лимите контейнера 512 MiB это разница между OOM и запасом.
|
||||
MALLOC_MMAP_THRESHOLD_: "1048576"
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
volumes:
|
||||
- ./secrets/pki/api:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [frontend, data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
# Временные файлы текущих загрузок (до 10 MiB каждый) — tmpfs, учитывается в лимите памяти контейнера.
|
||||
tmpfs: ["/tmp:size=160m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
janitor:
|
||||
build: *app-build
|
||||
command: ["python", "janitor.py"]
|
||||
env_file: .env
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *otel-env]
|
||||
BAO_ROLE: janitor
|
||||
DB_USER: messenger_janitor
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
BROKER_URL: ""
|
||||
OTEL_SERVICE_NAME: messenger-janitor
|
||||
volumes:
|
||||
- ./secrets/pki/janitor:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 192M, pids: 256}
|
||||
|
||||
dispatcher:
|
||||
build: *app-build
|
||||
command: ["python", "dispatcher.py"]
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
|
||||
BAO_ROLE: dispatcher
|
||||
DB_USER: messenger_dispatcher
|
||||
BROKER_URL: amqps://rabbitmq:5671/messenger # не секрет: вход по сертификату CN=dispatcher
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
volumes:
|
||||
- ./secrets/pki/dispatcher:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- faults:/faults
|
||||
networks: [data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||||
|
||||
# Worker: обработка изображения в дочернем процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB,
|
||||
# RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS; контейнер ограничен по CPU/RAM/PID.
|
||||
worker:
|
||||
build: *app-build
|
||||
command: ["python", "worker.py"]
|
||||
environment:
|
||||
<<: [*tls-env, *db-env, *pipeline-env, *otel-env]
|
||||
BAO_ROLE: worker
|
||||
DB_USER: messenger_worker
|
||||
BROKER_URL: amqps://rabbitmq:5671/messenger # вход по сертификату CN=worker
|
||||
S3_ENDPOINT: https://s3:8333
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
|
||||
WORKER_TMP_DIR: /tmp/work
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
volumes:
|
||||
- ./secrets/pki/worker:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- faults:/faults
|
||||
networks: [data, secrets, telemetry]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
openbao: *bao-ready
|
||||
otel-gate: *otel-depends
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=256m,mode=1777"]
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 70s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "2.00", memory: 1536M, pids: 128}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.25-alpine
|
||||
ports:
|
||||
- "127.0.0.1:8443:8443"
|
||||
- "127.0.0.1:8404:8404"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/pki/ingress:/run/tls-ingress:ro
|
||||
- ./secrets/pki/lb:/run/tls-lb:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks: [ingress, frontend]
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M, pids: 256}
|
||||
|
||||
# ---------------- наблюдаемость ----------------
|
||||
# otel-gate владеет network namespace; Collector живёт в нём же и слушает только 127.0.0.1.
|
||||
# Имя otel-collector в сети telemetry указывает на gate (alias): приложение ходит на https://otel-collector:4318.
|
||||
otel-gate:
|
||||
image: haproxy:3.2.25-alpine
|
||||
volumes:
|
||||
- ./infra/otel/gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/pki/otel-collector:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
networks:
|
||||
telemetry:
|
||||
aliases: [otel-collector]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=8m,mode=1777", "/var/lib/haproxy:size=1m,uid=99,gid=99,mode=0750"]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8405/gate-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 64M, pids: 256}
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.164.0
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
network_mode: "service:otel-gate"
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
- ./secrets/pki/otel-collector:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
depends_on:
|
||||
otel-gate: {condition: service_started}
|
||||
<<: *hardening
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 384M, pids: 256}
|
||||
|
||||
# Одноразовый chown volumes Loki/Tempo для uid 10001 (их процессы — не root). Исключение: root + CAP_CHOWN,
|
||||
# без сети, завершается сразу. Идемпотентно: при повторном up каталоги уже 10001 и внутри — 0700-каталоги
|
||||
# Loki/Tempo, в которые root без CAP_DAC_READ_SEARCH не войдёт, поэтому chown только для ещё чужого volume.
|
||||
telemetry-init:
|
||||
image: busybox:1.37.0
|
||||
user: root
|
||||
command: ["sh", "-c", "for d in /loki /tempo; do [ \"$$(stat -c %u $$d)\" = 10001 ] || chown -R 10001:10001 $$d || exit 1; done"]
|
||||
volumes:
|
||||
- lokidata:/loki
|
||||
- tempodata:/tempo
|
||||
network_mode: none
|
||||
cap_drop: [ALL]
|
||||
cap_add: [CHOWN]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 32M, pids: 256}
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.15.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --web.config.file=/etc/prometheus/web.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=1GB
|
||||
- --web.enable-otlp-receiver
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro
|
||||
- ./infra/prometheus/promtool-http.yml:/etc/prometheus/promtool-http.yml:ro
|
||||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- ./infra/prometheus/tests:/etc/prometheus/tests:ro
|
||||
- ./secrets/pki/prometheus:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- promdata:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090"
|
||||
networks: [telemetry, ops]
|
||||
<<: *hardening
|
||||
# promtool test rules (unit-тесты алертов, check_configs.py) создаёт временную TSDB в /tmp
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD", "promtool", "check", "ready", "--url=https://localhost:9090", "--http.config.file=/etc/prometheus/promtool-http.yml"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.9
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- ./secrets/pki/loki:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- lokidata:/loki
|
||||
ports:
|
||||
- "127.0.0.1:3100:3100"
|
||||
networks: [telemetry, ops]
|
||||
depends_on:
|
||||
telemetry-init: {condition: service_completed_successfully}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:3.1.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- ./secrets/pki/tempo:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- tempodata:/var/tempo
|
||||
ports:
|
||||
- "127.0.0.1:3200:3200"
|
||||
networks: [telemetry, ops]
|
||||
depends_on:
|
||||
telemetry-init: {condition: service_completed_successfully}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.4.14
|
||||
environment:
|
||||
GF_SERVER_PROTOCOL: https
|
||||
GF_SERVER_CERT_FILE: /run/tls/tls.crt
|
||||
GF_SERVER_CERT_KEY: /run/tls/tls.key
|
||||
GF_SERVER_DOMAIN: localhost
|
||||
GF_SERVER_ROOT_URL: https://localhost:3000/
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_SECURITY_COOKIE_SECURE: "true"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
|
||||
secrets: [grafana_admin_password]
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
|
||||
- ./secrets/pki/grafana:/run/tls:ro
|
||||
- ./secrets/pki/public:/run/tls-ca:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
networks: [telemetry, ops]
|
||||
depends_on:
|
||||
prometheus: {condition: service_healthy}
|
||||
loki: {condition: service_started}
|
||||
tempo: {condition: service_started}
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=64m,mode=1777"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS -m 2 -o /dev/null --cacert /run/tls-ca/ca.crt https://localhost:3000/api/health || wget -q -T 2 -O /dev/null https://localhost:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 384M, pids: 256}
|
||||
|
||||
# ---------------- только для проверок (профиль tools, сам не стартует) ----------------
|
||||
# Контейнер-зонд во всех внутренних сетях: отрицательные/положительные проверки trust matrix изнутри
|
||||
# (student-tests/tls_probe.py). Получает ТОЛЬКО тестовые сертификаты и сертификаты клиентов, которыми
|
||||
# пробует войти «не туда»; ключа CA и секретов OpenBao у него нет.
|
||||
probe:
|
||||
build: *app-build
|
||||
profiles: [tools]
|
||||
command: ["python", "/probe/tls_probe.py", "matrix"]
|
||||
environment:
|
||||
PKI: /pki
|
||||
PYTHONPATH: /app
|
||||
volumes:
|
||||
- ./student-tests:/probe:ro
|
||||
- ./secrets/pki/public:/pki/public:ro
|
||||
- ./secrets/pki/test:/pki/test:ro
|
||||
- ./secrets/pki/lb:/pki/lb:ro
|
||||
- ./secrets/pki/api:/pki/api:ro
|
||||
- ./secrets/pki/worker:/pki/worker:ro
|
||||
- ./secrets/pki/dispatcher:/pki/dispatcher:ro
|
||||
- ./secrets/pki/janitor:/pki/janitor:ro
|
||||
- ./secrets/pki/grafana:/pki/grafana:ro
|
||||
- ./secrets/pki/otel-collector:/pki/otel-collector:ro
|
||||
- ./secrets/pki/operator:/pki/operator:ro
|
||||
networks: [frontend, data, secrets, telemetry]
|
||||
<<: *hardening
|
||||
tmpfs: ["/tmp:size=16m,mode=1777"]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||||
|
||||
networks:
|
||||
ingress: {}
|
||||
frontend:
|
||||
internal: true
|
||||
data:
|
||||
internal: true
|
||||
secrets:
|
||||
internal: true
|
||||
telemetry:
|
||||
internal: true
|
||||
# Только для публикации админ-интерфейсов на 127.0.0.1 (internal-сеть не может публиковать порты).
|
||||
ops: {}
|
||||
|
||||
volumes:
|
||||
openbaodata: {}
|
||||
pgdata: {}
|
||||
redisdata: {}
|
||||
s3data: {}
|
||||
rabbitmqdata: {}
|
||||
faults: {}
|
||||
promdata: {}
|
||||
lokidata: {}
|
||||
tempodata: {}
|
||||
grafanadata: {}
|
||||
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
s3_config: {file: ./secrets/s3_config.json}
|
||||
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
|
||||
grafana_admin_password: {file: ./secrets/grafana_admin_password}
|
||||
Reference in New Issue
Block a user