forked from mtusi2026/mtusi2026lab6
338 lines
13 KiB
YAML
338 lines
13 KiB
YAML
# Стенд lab6: HAProxy -> API x N -> PostgreSQL (метаданные) + SeaweedFS (приватный S3, байты) + Valkey (кэш сессий).
|
|
# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). Запуск и проверки — см. REPORT.md.
|
|
name: mtusi-lab6
|
|
|
|
x-common: &common
|
|
restart: unless-stopped
|
|
logging:
|
|
driver: json-file
|
|
options: {max-size: "10m", max-file: "3"}
|
|
|
|
services:
|
|
postgres:
|
|
<<: *common
|
|
image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24
|
|
environment:
|
|
POSTGRES_USER: messenger
|
|
POSTGRES_DB: messenger
|
|
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
|
secrets: [postgres_password]
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"]
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 12
|
|
mem_limit: 512m
|
|
cpus: ${POSTGRES_CPUS:-1.0}
|
|
networks: [backend]
|
|
|
|
valkey:
|
|
<<: *common
|
|
image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
|
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec",
|
|
"--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""]
|
|
volumes:
|
|
- valkeydata:/data
|
|
healthcheck:
|
|
test: ["CMD", "valkey-cli", "ping"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 12
|
|
mem_limit: 128m
|
|
cpus: 0.5
|
|
networks: [backend]
|
|
|
|
# Приватное S3-совместимое хранилище SeaweedFS (all-in-one: master + volume + filer + S3-шлюз).
|
|
# Порты наружу не публикуются: доступ только из сети backend и только по ключам из s3.json (identities).
|
|
# Анонимный доступ запрещён (при заданных identities запрос без подписи получает 403).
|
|
seaweedfs:
|
|
<<: *common
|
|
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
|
|
entrypoint: ["/bin/sh", "-c"]
|
|
command: ["exec weed server -dir=/data -ip=seaweedfs -master.volumeSizeLimitMB=512 -volume.max=30 -filer -s3 -s3.port=8333 -s3.config=/run/secrets/seaweed_s3_config"]
|
|
secrets: [seaweed_s3_config]
|
|
volumes:
|
|
- seaweeddata:/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && nc -z 127.0.0.1 8333"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 24
|
|
start_period: 10s
|
|
mem_limit: 768m
|
|
cpus: ${S3_CPUS:-1.0}
|
|
networks: [backend]
|
|
|
|
# Идемпотентный bootstrap: bucket создаётся, если его нет; проверяется, что анонимный доступ запрещён и что ключ API работает.
|
|
s3-init:
|
|
build: .
|
|
image: mtusi-messenger:lab6
|
|
command: ["python", "-m", "app.s3_bootstrap"]
|
|
environment:
|
|
S3_ENDPOINT: http://seaweedfs:8333
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key
|
|
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key
|
|
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
|
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
|
secrets: [s3_admin_access_key, s3_admin_secret_key, s3_access_key, s3_secret_key]
|
|
depends_on:
|
|
seaweedfs: {condition: service_healthy}
|
|
restart: "no"
|
|
networks: [backend]
|
|
|
|
# Брокер: durable quorum-очередь, persistent-сообщения, publisher confirms; данные на volume rabbitmqdata.
|
|
# hostname фиксирован: имя узла RabbitMQ определяет каталог данных, иначе очередь не переживёт пересоздание контейнера.
|
|
rabbitmq:
|
|
<<: *common
|
|
image: rabbitmq:4.1-alpine@sha256:6a0f7984185cd2cdbf78a184b4c17abe88ecbbda6006a7acf17dbc7e4307f4e2
|
|
hostname: rabbitmq
|
|
# RabbitMQ 4.x: RABBITMQ_DEFAULT_USER/PASS объявлены устаревшими -> учётные данные в конфигурационном файле (secret)
|
|
secrets:
|
|
- source: rabbitmq_conf
|
|
target: /etc/rabbitmq/conf.d/20-credentials.conf
|
|
mode: 0444
|
|
volumes:
|
|
- rabbitmqdata:/var/lib/rabbitmq
|
|
healthcheck:
|
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
|
interval: 10s
|
|
timeout: 8s
|
|
retries: 12
|
|
mem_limit: 512m
|
|
cpus: ${RABBITMQ_CPUS:-1.0}
|
|
networks: [backend]
|
|
|
|
migrate:
|
|
build: .
|
|
image: mtusi-messenger:lab6
|
|
command: ["python", "-m", "app.migrate"]
|
|
environment:
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
secrets: [database_url]
|
|
depends_on:
|
|
postgres: {condition: service_healthy}
|
|
restart: "no"
|
|
networks: [backend]
|
|
|
|
api:
|
|
<<: *common
|
|
build: .
|
|
image: mtusi-messenger:lab6
|
|
environment:
|
|
APP_ENV: ${APP_ENV:-lab}
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
VALKEY_URL: redis://valkey:6379/0
|
|
S3_ENDPOINT: http://seaweedfs:8333
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
|
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
|
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
|
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
|
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
|
|
MAX_INFLIGHT: ${MAX_INFLIGHT:-64}
|
|
ACCESS_LOG: ${ACCESS_LOG:-1}
|
|
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
|
|
FAULT_INJECTION: ${API_FAULT_INJECTION:-}
|
|
BROKER_URL_FILE: /run/secrets/broker_url
|
|
MAX_QUEUED_JOBS: ${MAX_QUEUED_JOBS:-200}
|
|
SERVICE_NAME: messenger-api
|
|
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
|
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
|
OTEL_QUEUE_SIZE: ${OTEL_QUEUE_SIZE:-2048}
|
|
OTEL_BATCH_SIZE: ${OTEL_BATCH_SIZE:-512}
|
|
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
|
|
depends_on:
|
|
migrate: {condition: service_completed_successfully}
|
|
s3-init: {condition: service_completed_successfully}
|
|
valkey: {condition: service_healthy}
|
|
stop_grace_period: 15s
|
|
mem_limit: ${API_MEM:-256m}
|
|
cpus: ${API_CPUS:-1.0}
|
|
deploy:
|
|
replicas: ${API_REPLICAS:-2}
|
|
networks: [backend]
|
|
|
|
# Worker: отдельный процесс (тот же образ). Берёт задания из RabbitMQ, обрабатывает изображения, пишет результат в S3 и БД.
|
|
worker:
|
|
<<: *common
|
|
build: .
|
|
image: mtusi-messenger:lab6
|
|
command: ["python", "-m", "app.worker"]
|
|
environment:
|
|
SERVICE_NAME: messenger-worker
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
S3_ENDPOINT: http://seaweedfs:8333
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
|
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
|
BROKER_URL_FILE: /run/secrets/broker_url
|
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
|
MAX_ATTEMPTS: ${MAX_ATTEMPTS:-5}
|
|
LEASE_SECONDS: ${LEASE_SECONDS:-60}
|
|
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2}
|
|
FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-}
|
|
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
|
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
|
OTEL_QUEUE_SIZE: ${OTEL_QUEUE_SIZE:-2048}
|
|
OTEL_BATCH_SIZE: ${OTEL_BATCH_SIZE:-512}
|
|
ACCESS_LOG: "0"
|
|
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
|
|
depends_on:
|
|
migrate: {condition: service_completed_successfully}
|
|
s3-init: {condition: service_completed_successfully}
|
|
rabbitmq: {condition: service_healthy}
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker.alive') < 30 else 1)"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 20s
|
|
stop_grace_period: 30s
|
|
mem_limit: ${WORKER_MEM:-768m}
|
|
cpus: ${WORKER_CPUS:-1.0}
|
|
deploy:
|
|
replicas: ${WORKER_REPLICAS:-1}
|
|
networks: [backend]
|
|
|
|
# ------------------------------ наблюдаемость (lab6) ------------------------------
|
|
# OTLP-порты 4317/4318 и метрики Collector 8888/8889 доступны только внутри сети backend (наружу не публикуются).
|
|
otel-collector:
|
|
<<: *common
|
|
image: otel/opentelemetry-collector-contrib:0.161.0@sha256:fd328de2552466ad78385e1b1289c3f2402b1c45f265b252aab1955b42845ac1
|
|
command: ["--config=${COLLECTOR_CONFIG:-/etc/otelcol/collector.yaml}"]
|
|
volumes:
|
|
- ./infra/otel:/etc/otelcol:ro
|
|
mem_limit: 384m
|
|
cpus: ${COLLECTOR_CPUS:-1.0}
|
|
networks: [backend]
|
|
|
|
prometheus:
|
|
<<: *common
|
|
image: prom/prometheus:v3.15.0@sha256:efd719c99d83b060d9daefdcf00360461adf279f45ef5391f8d111892118753e
|
|
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d",
|
|
"--web.enable-lifecycle"]
|
|
volumes:
|
|
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
|
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
|
- promdata:/prometheus
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 6
|
|
mem_limit: 512m
|
|
cpus: 1.0
|
|
networks: [backend]
|
|
|
|
loki:
|
|
<<: *common
|
|
image: grafana/loki:3.7.8@sha256:1107dd5274e0ada47e42472b7a7e71f3b2a2fe878878108f3e2f9e51528f0193
|
|
command: ["-config.file=/etc/loki/loki.yaml"]
|
|
volumes:
|
|
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
|
- lokidata:/loki
|
|
mem_limit: 512m
|
|
cpus: 1.0
|
|
networks: [backend]
|
|
|
|
tempo:
|
|
<<: *common
|
|
image: grafana/tempo:2.9.0@sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4
|
|
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
|
volumes:
|
|
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
|
- tempodata:/var/tempo
|
|
mem_limit: 1g
|
|
cpus: 1.0
|
|
networks: [backend]
|
|
|
|
# Grafana: единственный опубликованный сервис наблюдаемости, только 127.0.0.1:3000. Datasources, dashboard и alert-вью
|
|
# подключаются provisioning-ом из Git (infra/grafana). Анонимный доступ выключен; пароль admin — secret.
|
|
grafana:
|
|
<<: *common
|
|
image: grafana/grafana:13.2.3@sha256:b28bae15e219c998fb0e0424ed724930cc61b1f61fb404d47c862f9a23f9e572
|
|
environment:
|
|
GF_SECURITY_ADMIN_USER: admin
|
|
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
|
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
|
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json
|
|
secrets: [grafana_admin_password]
|
|
volumes:
|
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
|
- grafanadata:/var/lib/grafana
|
|
depends_on: [prometheus, loki, tempo]
|
|
ports:
|
|
- "127.0.0.1:3000:3000"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"]
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 12
|
|
mem_limit: 512m
|
|
cpus: 1.0
|
|
networks: [backend]
|
|
|
|
lb:
|
|
<<: *common
|
|
image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6
|
|
volumes:
|
|
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
depends_on:
|
|
- api
|
|
ports:
|
|
- "127.0.0.1:8080:8080"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 6
|
|
mem_limit: 128m
|
|
cpus: ${LB_CPUS:-1.0}
|
|
stop_grace_period: 10s
|
|
networks: [backend]
|
|
|
|
secrets:
|
|
postgres_password:
|
|
file: ./secrets/postgres_password
|
|
database_url:
|
|
file: ./secrets/database_url
|
|
seaweed_s3_config:
|
|
file: ./secrets/seaweed_s3.json
|
|
s3_admin_access_key:
|
|
file: ./secrets/s3_admin_access_key
|
|
s3_admin_secret_key:
|
|
file: ./secrets/s3_admin_secret_key
|
|
s3_access_key:
|
|
file: ./secrets/s3_access_key
|
|
s3_secret_key:
|
|
file: ./secrets/s3_secret_key
|
|
rabbitmq_conf:
|
|
file: ./secrets/rabbitmq.conf
|
|
grafana_admin_password:
|
|
file: ./secrets/grafana_admin_password
|
|
broker_url:
|
|
file: ./secrets/broker_url
|
|
|
|
volumes:
|
|
pgdata:
|
|
valkeydata:
|
|
seaweeddata:
|
|
rabbitmqdata:
|
|
promdata:
|
|
lokidata:
|
|
tempodata:
|
|
grafanadata:
|
|
|
|
networks:
|
|
backend:
|