forked from mtusi2026/mtusi2026lab4
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
305 lines
9.4 KiB
YAML
305 lines
9.4 KiB
YAML
# Lab4: Client → HAProxy → API×N → PostgreSQL (metadata) + Redis (sessions)
|
||
# ↘ private S3 bucket (SeaweedFS, bytes only).
|
||
# Public entrypoint only via lb:8080. API replicas and S3 publish no host ports.
|
||
services:
|
||
postgres:
|
||
image: postgres:16.6-bookworm
|
||
environment:
|
||
POSTGRES_USER: messenger
|
||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||
POSTGRES_DB: messenger
|
||
secrets:
|
||
- postgres_password
|
||
volumes:
|
||
- pgdata:/var/lib/postgresql/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 10
|
||
start_period: 10s
|
||
restart: unless-stopped
|
||
mem_limit: 512m
|
||
cpus: 0.50
|
||
networks:
|
||
- labnet
|
||
|
||
redis:
|
||
image: redis:7.4.2-bookworm
|
||
command:
|
||
- redis-server
|
||
- --appendonly
|
||
- "yes"
|
||
- --appendfsync
|
||
- everysec
|
||
- --dir
|
||
- /data
|
||
volumes:
|
||
- redisdata:/data
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 10
|
||
start_period: 5s
|
||
restart: unless-stopped
|
||
mem_limit: 128m
|
||
cpus: 0.25
|
||
networks:
|
||
- labnet
|
||
|
||
# SeaweedFS all-in-one (master + volume + filer + S3 gateway). Every listener,
|
||
# including the master/filer web UIs and the unauthenticated gRPC ports, is bound
|
||
# to 127.0.0.1 inside this container; only s3-gw exposes the S3 HTTP API.
|
||
s3:
|
||
image: chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e
|
||
command:
|
||
- server
|
||
- -dir=/data
|
||
- -ip=127.0.0.1
|
||
- -ip.bind=127.0.0.1
|
||
- -s3
|
||
- -s3.port=8333
|
||
- -s3.ip.bind=127.0.0.1
|
||
- -s3.config=/run/secrets/s3_config
|
||
- -filer.exposeDirectoryData=false
|
||
- -master.volumeSizeLimitMB=512
|
||
- -master.telemetry=false
|
||
# 0 = free disk / volume size. A fixed count caps the bucket: with 16 slots
|
||
# (7 taken by the default collection) uploads failed after ≈4.7 GiB.
|
||
- -volume.max=0
|
||
environment:
|
||
# One weed process holds ≈30–40 MiB per in-flight 10 MiB PUT (s3 → filer → volume);
|
||
# API slots cap S3 at replicas × MAX_CONCURRENT_UPLOADS PUTs (16 → ≈500 MiB live).
|
||
# With 512m and no soft limit 10–40 parallel uploads OOM-killed weed (exit 137).
|
||
GOMEMLIMIT: 768MiB
|
||
secrets:
|
||
- s3_config
|
||
volumes:
|
||
- s3data:/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 20
|
||
start_period: 15s
|
||
stop_grace_period: 20s
|
||
restart: unless-stopped
|
||
mem_limit: 1g
|
||
cpus: 0.50
|
||
networks:
|
||
storage:
|
||
aliases:
|
||
- s3
|
||
|
||
# TCP sidecar in the s3 network namespace: publishes ONLY the S3 HTTP API as s3:9000.
|
||
# A restart of s3 gives it a new network namespace; the entrypoint then exits and
|
||
# `restart: unless-stopped` starts s3-gw again inside the new one (infra/s3-gw-entrypoint.sh).
|
||
# Compose-driven restarts/recreation of s3 also restart s3-gw (depends_on restart: true).
|
||
s3-gw:
|
||
image: haproxy:2.9.14-bookworm
|
||
network_mode: "service:s3"
|
||
entrypoint: ["/bin/sh", "/usr/local/bin/s3-gw-entrypoint.sh"]
|
||
stop_signal: SIGTERM
|
||
volumes:
|
||
- ./infra/s3-gateway.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||
- ./infra/s3-gw-entrypoint.sh:/usr/local/bin/s3-gw-entrypoint.sh:ro
|
||
depends_on:
|
||
s3:
|
||
condition: service_healthy
|
||
restart: true
|
||
healthcheck:
|
||
# Config valid AND still attached to the live namespace of s3 (not just `lo`).
|
||
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && grep -qv -e '^ *lo:' -e '|' /proc/net/dev"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 5
|
||
start_period: 5s
|
||
restart: unless-stopped
|
||
mem_limit: 64m
|
||
cpus: 0.25
|
||
|
||
# One-shot, idempotent bucket bootstrap with the S3 admin identity (API never gets it).
|
||
s3-init:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
image: mtusi-messenger:lab4
|
||
command: ["python", "-m", "messenger.s3bootstrap"]
|
||
environment:
|
||
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
|
||
S3_REGION: "${S3_REGION:-us-east-1}"
|
||
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
|
||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||
secrets:
|
||
- s3_admin_access_key_id
|
||
- s3_admin_secret_access_key
|
||
depends_on:
|
||
s3-gw:
|
||
condition: service_started
|
||
restart: "no"
|
||
dns_opt:
|
||
- ndots:0
|
||
- timeout:1
|
||
- attempts:1
|
||
mem_limit: 128m
|
||
cpus: 0.25
|
||
networks:
|
||
- storage
|
||
|
||
migrate:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
image: mtusi-messenger:lab4
|
||
command: ["python", "-m", "messenger.migrate"]
|
||
environment:
|
||
DATABASE_URL_FILE: /run/secrets/database_url
|
||
MIGRATIONS_DIR: /app/migrations
|
||
secrets:
|
||
- database_url
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
restart: "no"
|
||
mem_limit: 256m
|
||
cpus: 0.25
|
||
networks:
|
||
- labnet
|
||
|
||
api:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
image: mtusi-messenger:lab4
|
||
environment:
|
||
APP_HOST: "0.0.0.0"
|
||
APP_PORT: "8080"
|
||
APP_ENV: lab
|
||
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
|
||
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
|
||
SESSION_STORE: redis
|
||
DATABASE_URL_FILE: /run/secrets/database_url
|
||
CACHE_URL_FILE: /run/secrets/cache_url
|
||
MIGRATIONS_DIR: /app/migrations
|
||
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
|
||
S3_REGION: "${S3_REGION:-us-east-1}"
|
||
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
|
||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||
MAX_UPLOAD_BYTES: "${MAX_UPLOAD_BYTES:-10485760}"
|
||
MAX_IMAGE_PIXELS: "${MAX_IMAGE_PIXELS:-40000000}"
|
||
MAX_CONCURRENT_UPLOADS: "${MAX_CONCURRENT_UPLOADS:-8}"
|
||
UPLOAD_IDLE_TIMEOUT_SECONDS: "${UPLOAD_IDLE_TIMEOUT_SECONDS:-10}"
|
||
UPLOAD_DEADLINE_SECONDS: "${UPLOAD_DEADLINE_SECONDS:-120}"
|
||
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
|
||
RECONCILE_INTERVAL_SECONDS: "${RECONCILE_INTERVAL_SECONDS:-300}"
|
||
SHUTDOWN_DRAIN_SECONDS: "${SHUTDOWN_DRAIN_SECONDS:-3}"
|
||
SHUTDOWN_TIMEOUT_SECONDS: "${SHUTDOWN_TIMEOUT_SECONDS:-8}"
|
||
secrets:
|
||
- database_url
|
||
- cache_url
|
||
- s3_access_key_id
|
||
- s3_secret_access_key
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
s3-init:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
# No host ports: only reachable via labnet / HAProxy.
|
||
expose:
|
||
- "8080"
|
||
healthcheck:
|
||
test:
|
||
[
|
||
"CMD",
|
||
"python",
|
||
"-c",
|
||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
|
||
]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 12
|
||
start_period: 20s
|
||
# SIGTERM → drain 3s (ready 503, still serving until HAProxy inter 1s × fall 2 drops
|
||
# it) → in-flight requests ≤ 8s → lifespan shutdown ≤ 2s; 15s leaves margin before SIGKILL.
|
||
# Scripts that call `docker stop` directly use the same `-t 15`.
|
||
stop_grace_period: 15s
|
||
restart: unless-stopped
|
||
# While s3 is stopped its DNS alias is gone and a lookup takes ~8 s with defaults,
|
||
# i.e. >24 s over boto3 retries (HAProxy 504 instead of a fast API 503).
|
||
dns_opt:
|
||
- ndots:0
|
||
- timeout:1
|
||
- attempts:1
|
||
# 512m: one full JPEG decode per replica may need ≈310 MiB (40 Mpx progressive CMYK).
|
||
mem_limit: 512m
|
||
cpus: 0.50
|
||
# Socket buffers count against the cgroup; uncapped autotuning (MiB per connection)
|
||
# lets queued uploads push the replica towards its memory limit.
|
||
sysctls:
|
||
net.ipv4.tcp_rmem: "4096 131072 262144"
|
||
net.ipv4.tcp_wmem: "4096 16384 262144"
|
||
networks:
|
||
- labnet
|
||
- storage
|
||
|
||
lb:
|
||
image: haproxy:2.9.14-bookworm
|
||
volumes:
|
||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||
ports:
|
||
- "127.0.0.1:8080:8080"
|
||
depends_on:
|
||
api:
|
||
condition: service_healthy
|
||
healthcheck:
|
||
# Image has no curl/wget; validate config + process. Public readiness is /health/ready via LB.
|
||
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 5
|
||
start_period: 5s
|
||
restart: unless-stopped
|
||
mem_limit: 128m
|
||
cpus: 0.25
|
||
sysctls:
|
||
net.ipv4.tcp_rmem: "4096 131072 262144"
|
||
net.ipv4.tcp_wmem: "4096 16384 262144"
|
||
networks:
|
||
- labnet
|
||
|
||
secrets:
|
||
postgres_password:
|
||
file: ./secrets/postgres_password
|
||
database_url:
|
||
file: ./secrets/database_url
|
||
cache_url:
|
||
file: ./secrets/cache_url
|
||
s3_config:
|
||
file: ./secrets/s3_config.json
|
||
s3_admin_access_key_id:
|
||
file: ./secrets/s3_admin_access_key_id
|
||
s3_admin_secret_access_key:
|
||
file: ./secrets/s3_admin_secret_access_key
|
||
s3_access_key_id:
|
||
file: ./secrets/s3_access_key_id
|
||
s3_secret_access_key:
|
||
file: ./secrets/s3_secret_access_key
|
||
|
||
volumes:
|
||
pgdata:
|
||
redisdata:
|
||
s3data:
|
||
|
||
networks:
|
||
labnet:
|
||
driver: bridge
|
||
# Storage segment: no route outside Docker, only api and s3-init can reach s3:9000.
|
||
storage:
|
||
driver: bridge
|
||
internal: true
|