name: messenger-lab6 x-app-build: &app-build context: . args: APP_VERSION: ${APP_VERSION:-dev} # Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py. x-pipeline-env: &pipeline-env PIPELINE_VERSION: ${PIPELINE_VERSION:-1} WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} WORKER_PREFETCH: ${WORKER_PREFETCH:-1} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15} JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} # Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults, # доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0. FAULT_INJECTION: ${FAULT_INJECTION:-1} FAULTS_DIR: /faults # lab6: OpenTelemetry SDK всех процессов приложения -> OTel Collector (OTLP/HTTP, внутренняя сеть telemetry). # Имена переменных — стандартные OTEL_* (их читает SDK); service.name задаётся в коде процесса и дублируется # в OTEL_SERVICE_NAME у каждого сервиса. OTEL_SDK_DISABLED=true в .env выключает телеметрию целиком. x-otel-env: &otel-env OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf OTEL_EXPORTER_OTLP_TIMEOUT: "5" # секунды (Python SDK): экспорт не держит память дольше OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс OTEL_BSP_MAX_QUEUE_SIZE: "2048" # спанов в памяти процесса, лишние отбрасываются и считаются OTEL_BLRP_MAX_QUEUE_SIZE: "2048" # записей логов, так же OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" # otel.sdk.processor.*: очереди и потери SDK OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} x-otel-depends: &otel-depends # Порядок старта, но не жёсткая зависимость: без Collector приложение работает, телеметрия теряется. condition: service_started required: false x-alive-check: &alive-check # Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад. test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"] interval: 10s timeout: 3s retries: 3 start_period: 20s start_interval: 1s x-s3-api-secrets: &s3-api-secrets S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key services: migrate: build: *app-build command: ["python", "migrate.py"] environment: DATABASE_URL_FILE: /run/secrets/database_url secrets: [database_url] networks: [data] depends_on: postgres: {condition: service_healthy} healthcheck: disable: true restart: "no" deploy: resources: limits: {cpus: "0.50", memory: 128M} # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. s3: image: chrislusf/seaweedfs:3.80 entrypoint: ["/usr/bin/weed"] command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", "-master.volumeSizeLimitMB=1024", "-volume.max=0", "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] secrets: [s3_config] volumes: ["s3data:/data"] networks: [data] healthcheck: # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] interval: 5s timeout: 3s retries: 24 start_period: 15s start_interval: 1s restart: unless-stopped stop_grace_period: 30s deploy: resources: limits: {cpus: "1.00", memory: 512M} # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. s3-init: build: *app-build command: ["python", "s3_init.py"] environment: S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] networks: [data] depends_on: s3: {condition: service_healthy} healthcheck: disable: true restart: "no" deploy: resources: limits: {cpus: "0.25", memory: 128M} api: build: *app-build env_file: .env environment: SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL OTEL_SERVICE_NAME: messenger-api <<: [*s3-api-secrets, *otel-env] secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] networks: [edge, data, telemetry] depends_on: migrate: {condition: service_completed_successfully} s3-init: {condition: service_completed_successfully} redis: {condition: service_healthy} otel-collector: *otel-depends healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] interval: 5s timeout: 3s retries: 3 start_period: 30s start_interval: 1s restart: unless-stopped stop_grace_period: 15s deploy: replicas: ${API_REPLICAS:-2} resources: limits: {cpus: "1.00", memory: 512M} # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. janitor: build: *app-build command: ["python", "janitor.py"] env_file: .env environment: CACHE_URL_FILE: "" # janitor не ходит в Redis BROKER_URL_FILE: "" # и в брокер OTEL_SERVICE_NAME: messenger-janitor <<: [*s3-api-secrets, *otel-env] secrets: [database_url, s3_access_key_id, s3_secret_access_key] networks: [data, telemetry] depends_on: migrate: {condition: service_completed_successfully} s3-init: {condition: service_completed_successfully} otel-collector: *otel-depends healthcheck: *alive-check restart: unless-stopped stop_grace_period: 10s deploy: resources: limits: {cpus: "0.25", memory: 192M} # Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права — # в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте. # hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume. rabbitmq: image: rabbitmq:4.1.4-alpine hostname: rabbitmq volumes: - ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro - rabbitmqdata:/var/lib/rabbitmq secrets: [rabbitmq_definitions] networks: [data] healthcheck: # Узел запущен, и vhost с очередями загружен (а не только «порт открыт»). test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"] interval: 5s timeout: 10s retries: 24 start_period: 20s start_interval: 2s restart: unless-stopped stop_grace_period: 30s deploy: resources: limits: {cpus: "1.00", memory: 512M} # Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает # просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет. dispatcher: build: *app-build command: ["python", "dispatcher.py"] environment: DATABASE_URL_FILE: /run/secrets/database_url BROKER_URL_FILE: /run/secrets/broker_url_dispatcher OTEL_SERVICE_NAME: messenger-dispatcher <<: [*pipeline-env, *otel-env] secrets: [database_url, broker_url_dispatcher] volumes: ["faults:/faults"] networks: [data, telemetry] depends_on: migrate: {condition: service_completed_successfully} rabbitmq: {condition: service_healthy} otel-collector: *otel-depends healthcheck: *alive-check restart: unless-stopped stop_grace_period: 10s deploy: resources: limits: {cpus: "0.25", memory: 128M} # Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем # процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS. # Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker. worker: build: *app-build command: ["python", "worker.py"] environment: DATABASE_URL_FILE: /run/secrets/database_url BROKER_URL_FILE: /run/secrets/broker_url_worker MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512} OTEL_SERVICE_NAME: messenger-worker <<: [*pipeline-env, *s3-api-secrets, *otel-env] secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key] volumes: ["faults:/faults"] tmpfs: - /tmp:size=256m,mode=1777 networks: [data, telemetry] depends_on: migrate: {condition: service_completed_successfully} s3-init: {condition: service_completed_successfully} rabbitmq: {condition: service_healthy} otel-collector: *otel-depends healthcheck: *alive-check restart: unless-stopped # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят; # если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова. stop_grace_period: 70s deploy: resources: limits: {cpus: "2.00", memory: 1536M} lb: image: haproxy:3.2.25-alpine ports: - "127.0.0.1:8080:8080" - "127.0.0.1:8404:8404" volumes: - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro networks: [edge] depends_on: api: {condition: service_healthy} healthcheck: test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] interval: 5s timeout: 3s retries: 3 start_period: 10s start_interval: 1s restart: unless-stopped stop_grace_period: 15s deploy: resources: limits: {cpus: "0.50", memory: 128M} postgres: image: postgres:17.11-alpine environment: POSTGRES_USER: messenger POSTGRES_DB: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] secrets: [postgres_password] volumes: ["pgdata:/var/lib/postgresql/data"] networks: [data] healthcheck: test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] interval: 5s timeout: 3s retries: 12 start_period: 10s restart: unless-stopped stop_grace_period: 30s deploy: resources: limits: {cpus: "1.00", memory: 512M} redis: image: redis:7.4.11-alpine command: ["redis-server", "/usr/local/etc/redis/redis.conf"] volumes: - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro - redisdata:/data secrets: [redis_auth] networks: [data] healthcheck: test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] interval: 5s timeout: 3s retries: 12 restart: unless-stopped deploy: resources: limits: {cpus: "0.50", memory: 128M} # ---------------- lab6: наблюдаемость ---------------- # OTel Collector: приём OTLP от приложения, memory_limiter + batch, ограниченные очереди exporter, # отправка в Tempo / Loki / Prometheus. Не хранилище. Порты не публикуются (только сеть telemetry). # Образ distroless (без shell), поэтому без healthcheck: здоровье видно по otelcol_* в Prometheus. otel-collector: image: otel/opentelemetry-collector-contrib:0.164.0 command: ["--config=/etc/otelcol-contrib/config.yaml"] volumes: - ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro networks: [telemetry] restart: unless-stopped stop_grace_period: 10s deploy: resources: # heap ограничивает memory_limiter (200 MiB); сверху — runtime Go и отображённый бинарник contrib. limits: {cpus: "0.50", memory: 384M} # Одноразовый bootstrap прав на volumes Loki и Tempo (оба процесса работают как uid 10001, не root). telemetry-init: image: busybox:1.37.0 user: root command: ["sh", "-c", "chown -R 10001:10001 /loki /tempo"] volumes: - lokidata:/loki - tempodata:/tempo network_mode: none restart: "no" deploy: resources: limits: {cpus: "0.25", memory: 32M} # Prometheus: метрики (OTLP receiver для Collector + pull служебных метрик), recording/alert rules. # 127.0.0.1:9090 — только для evidence-скриптов и /alerts на защите. prometheus: image: prom/prometheus:v3.15.1 command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=7d - --storage.tsdb.retention.size=1GB - --web.enable-otlp-receiver volumes: - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./infra/prometheus/rules:/etc/prometheus/rules:ro - ./infra/prometheus/tests:/etc/prometheus/tests:ro # promtool test rules /etc/prometheus/tests/alerts_test.yml - promdata:/prometheus ports: - "127.0.0.1:9090:9090" networks: [telemetry, ops] healthcheck: test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] interval: 5s timeout: 3s retries: 24 start_period: 10s restart: unless-stopped stop_grace_period: 30s deploy: resources: limits: {cpus: "1.00", memory: 512M} # Loki: логи (OTLP от Collector), файловое хранилище в volume lokidata. 127.0.0.1:3100 — для тестов. loki: image: grafana/loki:3.7.9 command: ["-config.file=/etc/loki/loki.yaml"] volumes: - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro - lokidata:/loki ports: - "127.0.0.1:3100:3100" networks: [telemetry, ops] depends_on: telemetry-init: {condition: service_completed_successfully} restart: unless-stopped stop_grace_period: 30s deploy: resources: limits: {cpus: "1.00", memory: 512M} # Tempo: трассы (OTLP gRPC от Collector), блоки в volume tempodata. 127.0.0.1:3200 — для тестов. tempo: image: grafana/tempo:3.1.1 command: ["-config.file=/etc/tempo/tempo.yaml"] volumes: - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempodata:/var/tempo ports: - "127.0.0.1:3200:3200" networks: [telemetry, ops] depends_on: telemetry-init: {condition: service_completed_successfully} restart: unless-stopped stop_grace_period: 30s deploy: resources: limits: {cpus: "1.00", memory: 512M} # Grafana: интерфейс и запросы к трём хранилищам. Источники данных и dashboard — provisioning из Git # (infra/grafana), ручного набора нет. Вход: admin / пароль из secrets/grafana_admin_password. grafana: image: grafana/grafana:12.4.14 environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_ANONYMOUS_ENABLED: "false" GF_ANALYTICS_REPORTING_ENABLED: "false" GF_ANALYTICS_CHECK_FOR_UPDATES: "false" GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" GF_NEWS_NEWS_FEED_ENABLED: "false" GF_PLUGINS_PREINSTALL_DISABLED: "true" # стенд не скачивает плагины из интернета при старте GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json secrets: [grafana_admin_password] volumes: - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro - ./infra/grafana/dashboards:/etc/grafana/dashboards:ro - grafanadata:/var/lib/grafana ports: - "127.0.0.1:3000:3000" networks: [telemetry, ops] depends_on: prometheus: {condition: service_healthy} loki: {condition: service_started} tempo: {condition: service_started} healthcheck: # wget есть в образе на Alpine, curl — в варианте на Ubuntu: проверка работает в обоих. test: ["CMD-SHELL", "wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health || curl -fsS -m 2 -o /dev/null http://127.0.0.1:3000/api/health"] interval: 5s timeout: 3s retries: 24 start_period: 20s restart: unless-stopped stop_grace_period: 15s deploy: resources: limits: {cpus: "0.50", memory: 384M} networks: edge: {} data: internal: true # lab6: приложение -> Collector -> хранилища -> Grafana. Внутренняя: наружу ничего не публикуется. telemetry: internal: true # Только для публикации UI/API хранилищ на 127.0.0.1 (internal-сеть не может публиковать порты). ops: {} volumes: pgdata: {} redisdata: {} s3data: {} rabbitmqdata: {} faults: {} promdata: {} lokidata: {} tempodata: {} grafanadata: {} secrets: postgres_password: {file: ./secrets/postgres_password} database_url: {file: ./secrets/database_url} cache_url: {file: ./secrets/cache_url} redis_auth: {file: ./secrets/redis_auth} s3_root_user: {file: ./secrets/s3_root_user} s3_root_password: {file: ./secrets/s3_root_password} s3_access_key_id: {file: ./secrets/s3_access_key_id} s3_secret_access_key: {file: ./secrets/s3_secret_access_key} s3_config: {file: ./secrets/s3_config.json} broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher} broker_url_worker: {file: ./secrets/broker_url_worker} rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json} grafana_admin_password: {file: ./secrets/grafana_admin_password}