Files
mtusi2026lab4/compose.yaml
T
Mikhail Verkovykh 728e53cc55 Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
2026-11-06 19:48:37 +03:00

305 lines
9.4 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Lab4: Client → HAProxy → API×N → PostgreSQL (metadata) + Redis (sessions)
# ↘ private S3 bucket (SeaweedFS, bytes only).
# Public entrypoint only via lb:8080. API replicas and S3 publish no host ports.
services:
postgres:
image: postgres:16.6-bookworm
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
secrets:
- postgres_password
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
restart: unless-stopped
mem_limit: 512m
cpus: 0.50
networks:
- labnet
redis:
image: redis:7.4.2-bookworm
command:
- redis-server
- --appendonly
- "yes"
- --appendfsync
- everysec
- --dir
- /data
volumes:
- redisdata:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
start_period: 5s
restart: unless-stopped
mem_limit: 128m
cpus: 0.25
networks:
- labnet
# SeaweedFS all-in-one (master + volume + filer + S3 gateway). Every listener,
# including the master/filer web UIs and the unauthenticated gRPC ports, is bound
# to 127.0.0.1 inside this container; only s3-gw exposes the S3 HTTP API.
s3:
image: chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e
command:
- server
- -dir=/data
- -ip=127.0.0.1
- -ip.bind=127.0.0.1
- -s3
- -s3.port=8333
- -s3.ip.bind=127.0.0.1
- -s3.config=/run/secrets/s3_config
- -filer.exposeDirectoryData=false
- -master.volumeSizeLimitMB=512
- -master.telemetry=false
# 0 = free disk / volume size. A fixed count caps the bucket: with 16 slots
# (7 taken by the default collection) uploads failed after ≈4.7 GiB.
- -volume.max=0
environment:
# One weed process holds ≈30–40 MiB per in-flight 10 MiB PUT (s3 → filer → volume);
# API slots cap S3 at replicas × MAX_CONCURRENT_UPLOADS PUTs (16 → ≈500 MiB live).
# With 512m and no soft limit 10–40 parallel uploads OOM-killed weed (exit 137).
GOMEMLIMIT: 768MiB
secrets:
- s3_config
volumes:
- s3data:/data
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"]
interval: 5s
timeout: 3s
retries: 20
start_period: 15s
stop_grace_period: 20s
restart: unless-stopped
mem_limit: 1g
cpus: 0.50
networks:
storage:
aliases:
- s3
# TCP sidecar in the s3 network namespace: publishes ONLY the S3 HTTP API as s3:9000.
# A restart of s3 gives it a new network namespace; the entrypoint then exits and
# `restart: unless-stopped` starts s3-gw again inside the new one (infra/s3-gw-entrypoint.sh).
# Compose-driven restarts/recreation of s3 also restart s3-gw (depends_on restart: true).
s3-gw:
image: haproxy:2.9.14-bookworm
network_mode: "service:s3"
entrypoint: ["/bin/sh", "/usr/local/bin/s3-gw-entrypoint.sh"]
stop_signal: SIGTERM
volumes:
- ./infra/s3-gateway.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./infra/s3-gw-entrypoint.sh:/usr/local/bin/s3-gw-entrypoint.sh:ro
depends_on:
s3:
condition: service_healthy
restart: true
healthcheck:
# Config valid AND still attached to the live namespace of s3 (not just `lo`).
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && grep -qv -e '^ *lo:' -e '|' /proc/net/dev"]
interval: 10s
timeout: 3s
retries: 5
start_period: 5s
restart: unless-stopped
mem_limit: 64m
cpus: 0.25
# One-shot, idempotent bucket bootstrap with the S3 admin identity (API never gets it).
s3-init:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab4
command: ["python", "-m", "messenger.s3bootstrap"]
environment:
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
S3_REGION: "${S3_REGION:-us-east-1}"
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
depends_on:
s3-gw:
condition: service_started
restart: "no"
dns_opt:
- ndots:0
- timeout:1
- attempts:1
mem_limit: 128m
cpus: 0.25
networks:
- storage
migrate:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab4
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
MIGRATIONS_DIR: /app/migrations
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
restart: "no"
mem_limit: 256m
cpus: 0.25
networks:
- labnet
api:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab4
environment:
APP_HOST: "0.0.0.0"
APP_PORT: "8080"
APP_ENV: lab
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
SESSION_STORE: redis
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
MIGRATIONS_DIR: /app/migrations
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
S3_REGION: "${S3_REGION:-us-east-1}"
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: "${MAX_UPLOAD_BYTES:-10485760}"
MAX_IMAGE_PIXELS: "${MAX_IMAGE_PIXELS:-40000000}"
MAX_CONCURRENT_UPLOADS: "${MAX_CONCURRENT_UPLOADS:-8}"
UPLOAD_IDLE_TIMEOUT_SECONDS: "${UPLOAD_IDLE_TIMEOUT_SECONDS:-10}"
UPLOAD_DEADLINE_SECONDS: "${UPLOAD_DEADLINE_SECONDS:-120}"
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
RECONCILE_INTERVAL_SECONDS: "${RECONCILE_INTERVAL_SECONDS:-300}"
SHUTDOWN_DRAIN_SECONDS: "${SHUTDOWN_DRAIN_SECONDS:-3}"
SHUTDOWN_TIMEOUT_SECONDS: "${SHUTDOWN_TIMEOUT_SECONDS:-8}"
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
redis:
condition: service_healthy
# No host ports: only reachable via labnet / HAProxy.
expose:
- "8080"
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
]
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
# SIGTERM → drain 3s (ready 503, still serving until HAProxy inter 1s × fall 2 drops
# it) → in-flight requests ≤ 8s → lifespan shutdown ≤ 2s; 15s leaves margin before SIGKILL.
# Scripts that call `docker stop` directly use the same `-t 15`.
stop_grace_period: 15s
restart: unless-stopped
# While s3 is stopped its DNS alias is gone and a lookup takes ~8 s with defaults,
# i.e. >24 s over boto3 retries (HAProxy 504 instead of a fast API 503).
dns_opt:
- ndots:0
- timeout:1
- attempts:1
# 512m: one full JPEG decode per replica may need ≈310 MiB (40 Mpx progressive CMYK).
mem_limit: 512m
cpus: 0.50
# Socket buffers count against the cgroup; uncapped autotuning (MiB per connection)
# lets queued uploads push the replica towards its memory limit.
sysctls:
net.ipv4.tcp_rmem: "4096 131072 262144"
net.ipv4.tcp_wmem: "4096 16384 262144"
networks:
- labnet
- storage
lb:
image: haproxy:2.9.14-bookworm
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:8080:8080"
depends_on:
api:
condition: service_healthy
healthcheck:
# Image has no curl/wget; validate config + process. Public readiness is /health/ready via LB.
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 5
start_period: 5s
restart: unless-stopped
mem_limit: 128m
cpus: 0.25
sysctls:
net.ipv4.tcp_rmem: "4096 131072 262144"
net.ipv4.tcp_wmem: "4096 16384 262144"
networks:
- labnet
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_config:
file: ./secrets/s3_config.json
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata:
redisdata:
s3data:
networks:
labnet:
driver: bridge
# Storage segment: no route outside Docker, only api and s3-init can reach s3:9000.
storage:
driver: bridge
internal: true