Files
mtusi2026lab6/compose.yaml
T

495 lines
20 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
name: messenger-lab6
x-app-build: &app-build
context: .
args:
APP_VERSION: ${APP_VERSION:-dev}
# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py.
x-pipeline-env: &pipeline-env
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
# Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults,
# доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0.
FAULT_INJECTION: ${FAULT_INJECTION:-1}
FAULTS_DIR: /faults
# lab6: OpenTelemetry SDK всех процессов приложения -> OTel Collector (OTLP/HTTP, внутренняя сеть telemetry).
# Имена переменных — стандартные OTEL_* (их читает SDK); service.name задаётся в коде процесса и дублируется
# в OTEL_SERVICE_NAME у каждого сервиса. OTEL_SDK_DISABLED=true в .env выключает телеметрию целиком.
x-otel-env: &otel-env
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_EXPORTER_OTLP_TIMEOUT: "5" # секунды (Python SDK): экспорт не держит память дольше
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс
OTEL_BSP_MAX_QUEUE_SIZE: "2048" # спанов в памяти процесса, лишние отбрасываются и считаются
OTEL_BLRP_MAX_QUEUE_SIZE: "2048" # записей логов, так же
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" # otel.sdk.processor.*: очереди и потери SDK
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
x-otel-depends: &otel-depends
# Порядок старта, но не жёсткая зависимость: без Collector приложение работает, телеметрия теряется.
condition: service_started
required: false
x-alive-check: &alive-check
# Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад.
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
interval: 10s
timeout: 3s
retries: 3
start_period: 20s
start_interval: 1s
x-s3-api-secrets: &s3-api-secrets
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
services:
migrate:
build: *app-build
command: ["python", "migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
# S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений.
# Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config
# (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket.
# Анонимного доступа нет: identity "anonymous" в конфиге не объявлена.
s3:
image: chrislusf/seaweedfs:3.80
entrypoint: ["/usr/bin/weed"]
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0",
"-master.volumeSizeLimitMB=1024", "-volume.max=0",
"-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"]
secrets: [s3_config]
volumes: ["s3data:/data"]
networks: [data]
healthcheck:
# S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт»)
test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"]
interval: 5s
timeout: 3s
retries: 24
start_period: 15s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки.
s3-init:
build: *app-build
command: ["python", "s3_init.py"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
s3: {condition: service_healthy}
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 128M}
api:
build: *app-build
env_file: .env
environment:
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL
OTEL_SERVICE_NAME: messenger-api
<<: [*s3-api-secrets, *otel-env]
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [edge, data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
redis: {condition: service_healthy}
otel-collector: *otel-depends
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "1.00", memory: 512M}
# Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ.
janitor:
build: *app-build
command: ["python", "janitor.py"]
env_file: .env
environment:
CACHE_URL_FILE: "" # janitor не ходит в Redis
BROKER_URL_FILE: "" # и в брокер
OTEL_SERVICE_NAME: messenger-janitor
<<: [*s3-api-secrets, *otel-env]
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
otel-collector: *otel-depends
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 192M}
# Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права —
# в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте.
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
rabbitmq:
image: rabbitmq:4.1.4-alpine
hostname: rabbitmq
volumes:
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
- rabbitmqdata:/var/lib/rabbitmq
secrets: [rabbitmq_definitions]
networks: [data]
healthcheck:
# Узел запущен, и vhost с очередями загружен (а не только «порт открыт»).
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
interval: 5s
timeout: 10s
retries: 24
start_period: 20s
start_interval: 2s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает
# просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет.
dispatcher:
build: *app-build
command: ["python", "dispatcher.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url_dispatcher
OTEL_SERVICE_NAME: messenger-dispatcher
<<: [*pipeline-env, *otel-env]
secrets: [database_url, broker_url_dispatcher]
volumes: ["faults:/faults"]
networks: [data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
otel-collector: *otel-depends
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 128M}
# Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем
# процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS.
# Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker.
worker:
build: *app-build
command: ["python", "worker.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url_worker
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
OTEL_SERVICE_NAME: messenger-worker
<<: [*pipeline-env, *s3-api-secrets, *otel-env]
secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key]
volumes: ["faults:/faults"]
tmpfs:
- /tmp:size=256m,mode=1777
networks: [data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
otel-collector: *otel-depends
healthcheck: *alive-check
restart: unless-stopped
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят;
# если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова.
stop_grace_period: 70s
deploy:
resources:
limits: {cpus: "2.00", memory: 1536M}
lb:
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8404:8404"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks: [edge]
depends_on:
api: {condition: service_healthy}
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
interval: 5s
timeout: 3s
retries: 3
start_period: 10s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
postgres:
image: postgres:17.11-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"]
secrets: [postgres_password]
volumes: ["pgdata:/var/lib/postgresql/data"]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
redis:
image: redis:7.4.11-alpine
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
volumes:
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
- redisdata:/data
secrets: [redis_auth]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
# ---------------- lab6: наблюдаемость ----------------
# OTel Collector: приём OTLP от приложения, memory_limiter + batch, ограниченные очереди exporter,
# отправка в Tempo / Loki / Prometheus. Не хранилище. Порты не публикуются (только сеть telemetry).
# Образ distroless (без shell), поэтому без healthcheck: здоровье видно по otelcol_* в Prometheus.
otel-collector:
image: otel/opentelemetry-collector-contrib:0.164.0
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
networks: [telemetry]
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
# heap ограничивает memory_limiter (200 MiB); сверху — runtime Go и отображённый бинарник contrib.
limits: {cpus: "0.50", memory: 384M}
# Одноразовый bootstrap прав на volumes Loki и Tempo (оба процесса работают как uid 10001, не root).
telemetry-init:
image: busybox:1.37.0
user: root
command: ["sh", "-c", "chown -R 10001:10001 /loki /tempo"]
volumes:
- lokidata:/loki
- tempodata:/tempo
network_mode: none
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 32M}
# Prometheus: метрики (OTLP receiver для Collector + pull служебных метрик), recording/alert rules.
# 127.0.0.1:9090 — только для evidence-скриптов и /alerts на защите.
prometheus:
image: prom/prometheus:v3.15.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- ./infra/prometheus/tests:/etc/prometheus/tests:ro # promtool test rules /etc/prometheus/tests/alerts_test.yml
- promdata:/prometheus
ports:
- "127.0.0.1:9090:9090"
networks: [telemetry, ops]
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 24
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Loki: логи (OTLP от Collector), файловое хранилище в volume lokidata. 127.0.0.1:3100 — для тестов.
loki:
image: grafana/loki:3.7.9
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
ports:
- "127.0.0.1:3100:3100"
networks: [telemetry, ops]
depends_on:
telemetry-init: {condition: service_completed_successfully}
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Tempo: трассы (OTLP gRPC от Collector), блоки в volume tempodata. 127.0.0.1:3200 — для тестов.
tempo:
image: grafana/tempo:3.1.1
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
ports:
- "127.0.0.1:3200:3200"
networks: [telemetry, ops]
depends_on:
telemetry-init: {condition: service_completed_successfully}
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Grafana: интерфейс и запросы к трём хранилищам. Источники данных и dashboard — provisioning из Git
# (infra/grafana), ручного набора нет. Вход: admin / пароль из secrets/grafana_admin_password.
grafana:
image: grafana/grafana:12.4.14
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true" # стенд не скачивает плагины из интернета при старте
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
networks: [telemetry, ops]
depends_on:
prometheus: {condition: service_healthy}
loki: {condition: service_started}
tempo: {condition: service_started}
healthcheck:
# wget есть в образе на Alpine, curl — в варианте на Ubuntu: проверка работает в обоих.
test: ["CMD-SHELL", "wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health || curl -fsS -m 2 -o /dev/null http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 24
start_period: 20s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M}
networks:
edge: {}
data:
internal: true
# lab6: приложение -> Collector -> хранилища -> Grafana. Внутренняя: наружу ничего не публикуется.
telemetry:
internal: true
# Только для публикации UI/API хранилищ на 127.0.0.1 (internal-сеть не может публиковать порты).
ops: {}
volumes:
pgdata: {}
redisdata: {}
s3data: {}
rabbitmqdata: {}
faults: {}
promdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
redis_auth: {file: ./secrets/redis_auth}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
s3_config: {file: ./secrets/s3_config.json}
broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher}
broker_url_worker: {file: ./secrets/broker_url_worker}
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
grafana_admin_password: {file: ./secrets/grafana_admin_password}