name: messenger-lab7 x-app: &app build: . image: messenger-lab7-app:dev # уникальное имя: на общем Docker-host не перепутать с чужими стендами networks: [backend] read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] tmpfs: ["/tmp:size=64m,mode=1777"] ulimits: core: { soft: 0, hard: 0 } volumes: [./secrets/tls/service-ca:/run/tls:ro] logging: &logging driver: json-file options: { max-size: "10m", max-file: "3" } # stdout-логи ограничены; история логов — в Loki # OpenTelemetry SDK api и worker (src/telemetry.ts). service.name задаётся у каждого сервиса отдельно. # OTEL_SDK_DISABLED=true — приложение без telemetry (сравнение, отладка). x-otel-env: &otel-env OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} OTEL_PROPAGATORS: tracecontext # W3C traceparent/tracestate; baggage клиентов не принимается OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс; окна rate() от 1 минуты OTEL_BSP_SCHEDULE_DELAY: "1000" OTEL_BLRP_SCHEDULE_DELAY: "1000" OTEL_NODE_EXPERIMENTAL_SDK_METRICS: "true" # otel.sdk.*: сколько spans/логов SDK обработал и отбросил x-s3-env: &s3-env S3_ENDPOINT: https://s3:9000 S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: messenger-attachments NODE_EXTRA_CA_CERTS: /run/tls/ca.crt x-jobs-env: &jobs-env MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} PIPELINE_VERSION: ${PIPELINE_VERSION:-1} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-2000} JOB_REPUBLISH_SECONDS: ${JOB_REPUBLISH_SECONDS:-600} FAILPOINTS: ${FAILPOINTS:-0} # 1 только на тестовом стенде: файл failpoint-<точка>.png один раз роняет процесс services: postgres: image: postgres:18.6-alpine@sha256:10a5c69a83035ab5239f5620d708520e8b28f5aa1f2b20143e6024532edadfb5 entrypoint: ["/bin/sh", "/etc/lab/entrypoint.sh"] environment: POSTGRES_USER: messenger POSTGRES_DB: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password secrets: [postgres_password] volumes: - pgdata:/var/lib/postgresql - ./infra/postgres:/etc/lab:ro - ./secrets/tls/postgres:/run/identity:ro healthcheck: test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] interval: 2s timeout: 2s retries: 20 deploy: resources: limits: { cpus: "2", memory: 1g } stop_grace_period: 30s restart: unless-stopped security_opt: [no-new-privileges:true] networks: [backend] openbao: image: openbao/openbao:2.5.4@sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115 user: "100:1000" entrypoint: ["/bin/bao"] command: ["server", "-config=/etc/lab/server.hcl"] volumes: - ./infra/openbao/server.hcl:/etc/lab/server.hcl:ro - ./secrets/tls/openbao:/run/identity:ro - bao-data:/openbao/file - bao-audit:/openbao/logs networks: [frontend, backend] ports: ["127.0.0.1:${BAO_PORT:-8200}:8200"] read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] tmpfs: ["/tmp:size=16m,mode=1777"] mem_limit: 256m cpus: 0.5 restart: unless-stopped # Filer, master и volume на loopback в том же контейнере. Наружу слушает только TLS S3. s3: image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", "-filer", "-s3", "-master.volumeSizeLimitMB=128", "-volume.minFreeSpace=64MiB", "-s3.ip.bind=0.0.0.0", "-s3.port=9000", "-s3.config=/run/s3/identities.json", "-s3.cert.file=/run/identity/tls.crt", "-s3.key.file=/run/identity/tls.key", "-s3.iam=false", "-s3.port.iceberg=0", "-s3.port.lance=0", "-s3.autoCreateBucket=false", "-s3.concurrentUploadLimitMB=256"] volumes: - s3data:/data - ./secrets/runtime/s3:/run/s3:ro - ./secrets/tls/s3:/run/identity:ro - ./secrets/tls/service-ca:/run/tls:ro healthcheck: test: ["CMD-SHELL", "SSL_CERT_FILE=/run/tls/ca.crt wget -qO /dev/null https://s3:9000/healthz"] interval: 2s timeout: 2s retries: 30 mem_limit: 768m cpus: 1 restart: unless-stopped security_opt: [no-new-privileges:true] networks: [backend] # Одноразовые сервисы до старта реплик: схема БД (advisory lock) и bucket (повторный запуск ничего не меняет). migrate: <<: *app command: ["node", "src/main.ts", "migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url secrets: [database_url] restart: "no" deploy: resources: limits: { cpus: "0.5", memory: 128m } depends_on: postgres: { condition: service_healthy } s3-init: <<: *app command: ["node", "src/main.ts", "s3-init"] environment: <<: *s3-env S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key secrets: - { source: s3_admin_access_key_id, target: s3_access_key_id } - { source: s3_admin_secret_access_key, target: s3_secret_access_key } restart: "no" deploy: resources: limits: { cpus: "0.5", memory: 128m } depends_on: s3: { condition: service_healthy } # Брокер: RabbitMQ, quorum-очередь на диске (volume rabbitmq-data), publisher confirms, ручной ack. # hostname фиксирует имя узла rabbit@rabbitmq: данные в volume привязаны к нему и переживают пересоздание. rabbitmq: image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5 hostname: rabbitmq volumes: - rabbitmq-data:/var/lib/rabbitmq - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro - ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro - ./secrets/tls/rabbitmq:/run/identity:ro - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/telemetry-ca:/run/telemetry:ro secrets: [rabbitmq_definitions] healthcheck: test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] interval: 5s timeout: 10s retries: 30 start_period: 20s deploy: resources: limits: { cpus: "1", memory: 512m } stop_grace_period: 30s restart: unless-stopped networks: [backend] api: <<: *app environment: <<: [*s3-env, *jobs-env, *otel-env] OTEL_SERVICE_NAME: messenger-api BAO_ROLE: api ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456} ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2} ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} DB_POOL_MAX: ${DB_POOL_MAX:-8} SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-2500} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} volumes: - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/api:/run/identity:ro - ./secrets/pki:/run/pki:ro depends_on: migrate: { condition: service_completed_successfully } s3-init: { condition: service_completed_successfully } healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://127.0.0.1:8081/"] interval: 2s timeout: 2s retries: 3 # Lab6: старт с OpenTelemetry SDK (загрузка instrumentations через ESM-hook) на занятом хосте — до ~25 с. # Успешная проверка сразу делает контейнер healthy; start_period только прощает неудачи при старте. start_period: 120s deploy: replicas: ${API_REPLICAS:-2} resources: limits: { cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-256m}" } stop_grace_period: 15s restart: unless-stopped # упавшая реплика поднимается сама; после `docker stop` остаётся остановленной # Worker: отдельный процесс без HTTP, конкурирующие consumers одной очереди. WORKER_CONCURRENCY = prefetch = # одновременных заданий на реплику; память ограничена контейнером, libvips — один поток, без кэша. worker: <<: *app command: ["node", "--import", "./src/telemetry.ts", "src/main.ts", "worker"] environment: <<: [*s3-env, *jobs-env, *otel-env] OTEL_SERVICE_NAME: messenger-worker BAO_ROLE: worker WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} volumes: - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/worker:/run/identity:ro depends_on: migrate: { condition: service_completed_successfully } s3-init: { condition: service_completed_successfully } rabbitmq: { condition: service_healthy } deploy: replicas: ${WORKER_REPLICAS:-2} resources: limits: { cpus: "${WORKER_CPUS:-1}", memory: "${WORKER_MEMORY:-768m}" } stop_grace_period: 15s # SIGTERM: consumer отменяется, текущие задания до 10 с, затем выход restart: unless-stopped # Периодическая сверка БД ↔ S3 с grace period (src/reconcile.ts). Свои S3-ключи: объекты + листинг bucket. reconciler: <<: *app command: ["node", "src/main.ts", "reconcile", "--apply", "--every=${RECONCILE_INTERVAL_SECONDS:-300}"] environment: <<: *s3-env BAO_ROLE: reconciler RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} volumes: - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/reconciler:/run/identity:ro depends_on: migrate: { condition: service_completed_successfully } s3-init: { condition: service_completed_successfully } deploy: resources: limits: { cpus: "0.5", memory: 128m } restart: unless-stopped lb: image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro - ./secrets/tls/ingress:/run/public:ro - ./secrets/tls/lb-client:/run/client:ro - ./secrets/tls/lb:/run/identity:ro - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/telemetry-ca:/run/telemetry:ro ports: ["127.0.0.1:${LB_PORT:-8443}:8443"] read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] depends_on: api: { condition: service_healthy } deploy: resources: limits: { cpus: "${LB_CPUS:-1}", memory: 128m } restart: unless-stopped networks: [frontend, backend] # HAProxy проверяет CN и CRL перед передачей OTLP по loopback в общем network namespace. otel-collector: image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b volumes: - ./infra/otel/ingress.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro - ./secrets/tls/otel-collector:/run/identity:ro - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/pki:/run/pki:ro networks: [backend, telemetry] read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] mem_limit: 128m cpus: 0.5 restart: unless-stopped logging: *logging collector: image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849 network_mode: "service:otel-collector" volumes: - ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/collector:/run/client:ro environment: GOMEMLIMIT: 180MiB read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] mem_limit: 256m cpus: 1 restart: unless-stopped logging: *logging prometheus: image: prom/prometheus:v3.15.1@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2 command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=15d - --storage.tsdb.retention.size=1GB - --web.enable-otlp-receiver - --web.config.file=/etc/prometheus/web.yml - --enable-feature=created-timestamp-zero-ingestion # новая серия счётчика начинается с 0: rate() видит первые ошибки - --query.lookback-delta=1m # данные остановленной реплики перестают учитываться через минуту, а не через 5 volumes: - ./infra/prometheus:/etc/prometheus:ro - ./secrets/tls/prometheus:/run/identity:ro - ./secrets/tls/telemetry-ca:/run/telemetry:ro - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/prometheus-scrape:/run/client:ro - prometheus-data:/prometheus deploy: resources: limits: { cpus: "1", memory: 512m } stop_grace_period: 30s restart: unless-stopped logging: *logging networks: [telemetry] loki: image: grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28 command: ["-config.file=/etc/loki/loki.yaml"] volumes: - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro - ./secrets/tls/loki:/run/identity:ro - ./secrets/tls/telemetry-ca:/run/telemetry:ro - loki-data:/loki deploy: resources: limits: { cpus: "1", memory: 384m } stop_grace_period: 30s restart: unless-stopped logging: *logging networks: [telemetry] tempo: image: grafana/tempo:2.10.10@sha256:12b0820f13ffa10ce956b6e9acc3b03f3cffded99109be6cc09d106fe56e5d38 command: ["-config.file=/etc/tempo/tempo.yaml"] volumes: - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - ./secrets/tls/tempo:/run/identity:ro - ./secrets/tls/telemetry-ca:/run/telemetry:ro - ./secrets/tls/collector-ca:/run/collector-ca:ro - tempo-data:/var/tempo deploy: resources: limits: { cpus: "1", memory: 512m } stop_grace_period: 30s restart: unless-stopped logging: *logging networks: [telemetry] # Grafana: интерфейс и запросы к хранилищам; datasources и dashboard импортируются из Git (provisioning). grafana: image: grafana/grafana:13.2.5@sha256:9f7aa1e1e3246aade5792d35a738049f7f32f89cd51d1956cf426a4e86268a84 entrypoint: - /bin/sh - -ec - | export DS_CA="$$(cat /run/tls/ca.crt)" DS_CERT="$$(cat /run/client/tls.crt)" DS_KEY="$$(cat /run/client/tls.key)" exec /run.sh environment: GF_SERVER_PROTOCOL: https GF_SERVER_CERT_FILE: /run/identity/tls.crt GF_SERVER_CERT_KEY: /run/identity/tls.key GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password GF_USERS_ALLOW_SIGN_UP: "false" GF_AUTH_ANONYMOUS_ENABLED: "false" GF_SERVER_ROOT_URL: https://localhost:${GRAFANA_PORT:-3000} GF_ANALYTICS_REPORTING_ENABLED: "false" GF_ANALYTICS_CHECK_FOR_UPDATES: "false" GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" GF_PLUGINS_PREINSTALL_DISABLED: "true" GF_NEWS_NEWS_FEED_ENABLED: "false" secrets: [grafana_admin_password] volumes: - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro - ./secrets/tls/grafana:/run/identity:ro - ./secrets/tls/service-ca:/run/tls:ro - ./secrets/tls/grafana-query:/run/client:ro - grafana-data:/var/lib/grafana ports: - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" healthcheck: test: ["CMD-SHELL", "SSL_CERT_FILE=/run/tls/ca.crt wget -qO /dev/null https://localhost:3000/api/health"] interval: 5s timeout: 3s retries: 30 deploy: resources: limits: { cpus: "0.5", memory: 256m } restart: unless-stopped logging: *logging networks: [frontend, telemetry] networks: frontend: {} backend: { internal: true } # api, worker, postgres, rabbitmq и s3-шлюз недоступны с хоста и не имеют выхода наружу telemetry: { internal: true } # collector → prometheus/loki/tempo ← grafana; приложение сюда не подключено volumes: bao-data: {} bao-audit: {} pgdata: {} s3data: {} rabbitmq-data: {} prometheus-data: {} loki-data: {} tempo-data: {} grafana-data: {} secrets: database_url: { file: ./secrets/runtime/database_url } postgres_password: { file: ./secrets/runtime/postgres_password } s3_admin_access_key_id: { file: ./secrets/runtime/s3_admin_access_key_id } s3_admin_secret_access_key: { file: ./secrets/runtime/s3_admin_secret_access_key } rabbitmq_definitions: { file: ./secrets/runtime/rabbitmq_definitions.json } grafana_admin_password: { file: ./secrets/runtime/grafana_admin_password }