upload_memory.py recreates the replicas and the balancer for every run of 10, 20 and 40 concurrent uploads of 1 and 10 MiB with 64 slots (effectively no limit) and with 3, drives them through lab4_seed.py from the prepared tooling, and keeps per-run JSON plus a summary with the cgroup memory.peak of each replica and of the balancer and the oom_kill counters. The seed script now also records memory.current before the load and reads X-Instance-Id regardless of case. The first run showed that slots alone only move a waiting body out of the worker: with 3 slots and 40 uploads of 10 MiB the kernel grew the socket buffers of HAProxy to about 110 MB and the 128 MiB balancer was killed. HAProxy now caps its client and server socket buffers at 64 KiB and the API replicas cap tcp_rmem and tcp_wmem at 256 KiB, which keeps the balancer at 30 MiB. glibc raised its mmap threshold after the first large free and kept freed 10 MiB bodies in the heap, so the API fixes MALLOC_MMAP_THRESHOLD_ at 128 KiB. With these settings 40 concurrent uploads of 10 MiB peak at 323 MiB of 512 per replica and all 80 succeed, while without the slot limit the same load reaches 512 MiB, a worker is killed and only 31 of 80 succeed. UPLOAD_CONCURRENCY stays at 3. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
277 lines
8.3 KiB
YAML
277 lines
8.3 KiB
YAML
x-app: &app
|
|
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
|
networks: [backend]
|
|
|
|
x-s3-env: &s3-env
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
|
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1}
|
|
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2}
|
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
|
IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2}
|
|
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-3}
|
|
UPLOAD_GRACE_SECONDS: ${UPLOAD_GRACE_SECONDS:-900}
|
|
UPLOAD_SWEEP_INTERVAL_SECONDS: ${UPLOAD_SWEEP_INTERVAL_SECONDS:-30}
|
|
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
|
|
|
|
x-s3-client: &s3-client
|
|
image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699
|
|
read_only: true
|
|
tmpfs: ["/tmp:size=16m"]
|
|
environment:
|
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
RC_CONFIG_DIR: /tmp/rc
|
|
volumes: ["./infra/s3:/s3:ro"]
|
|
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
|
cap_drop: [ALL]
|
|
security_opt: ["no-new-privileges:true"]
|
|
restart: "no"
|
|
networks: [backend]
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "0.5", memory: 64m}
|
|
|
|
services:
|
|
lb:
|
|
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
|
|
environment:
|
|
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
|
|
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
|
ports:
|
|
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
|
|
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
|
networks: [edge, backend]
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
|
|
interval: 2s
|
|
timeout: 2s
|
|
retries: 3
|
|
start_period: 10s
|
|
sysctls:
|
|
net.ipv4.ip_local_port_range: "10240 65535"
|
|
net.ipv4.tcp_tw_reuse: "1"
|
|
stop_signal: SIGUSR1
|
|
stop_grace_period: 8s
|
|
restart: unless-stopped
|
|
read_only: true
|
|
cap_drop: [ALL]
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "${LB_CPUS:-1}", memory: 128m}
|
|
|
|
api:
|
|
<<: *app
|
|
build: .
|
|
environment:
|
|
<<: *s3-env
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
CACHE_URL_FILE: /run/secrets/cache_url
|
|
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
|
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
|
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
|
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
|
|
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
|
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
|
|
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
|
|
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
|
|
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
|
|
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
|
|
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
|
MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072}
|
|
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 3
|
|
start_period: 30s
|
|
start_interval: 1s
|
|
stop_grace_period: 8s
|
|
restart: unless-stopped
|
|
read_only: true
|
|
tmpfs: [/tmp]
|
|
sysctls:
|
|
net.ipv4.tcp_rmem: "4096 65536 262144"
|
|
net.ipv4.tcp_wmem: "4096 65536 262144"
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
valkey:
|
|
condition: service_started
|
|
s3-init:
|
|
condition: service_completed_successfully
|
|
deploy:
|
|
replicas: ${API_REPLICAS:-2}
|
|
resources:
|
|
limits: {cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-512m}"}
|
|
|
|
migrate:
|
|
<<: *app
|
|
pull_policy: never
|
|
command: ["python", "-m", "messenger.migrate"]
|
|
environment:
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
|
secrets: [database_url]
|
|
healthcheck:
|
|
disable: true
|
|
restart: "no"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "0.5", memory: 256m}
|
|
|
|
postgres:
|
|
image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b
|
|
user: "70:70"
|
|
command:
|
|
- postgres
|
|
- -c
|
|
- max_connections=${PG_MAX_CONNECTIONS:-200}
|
|
- -c
|
|
- shared_buffers=${PG_SHARED_BUFFERS:-256MB}
|
|
- -c
|
|
- shared_preload_libraries=pg_stat_statements
|
|
- -c
|
|
- pg_stat_statements.track=top
|
|
- -c
|
|
- track_io_timing=on
|
|
- -c
|
|
- idle_in_transaction_session_timeout=10s
|
|
environment:
|
|
POSTGRES_DB: messenger
|
|
POSTGRES_USER: messenger
|
|
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
|
secrets: [postgres_password]
|
|
volumes: [pgdata:/var/lib/postgresql/data]
|
|
shm_size: 256m
|
|
healthcheck:
|
|
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 30
|
|
start_period: 10s
|
|
stop_grace_period: 10s
|
|
restart: unless-stopped
|
|
networks: [backend]
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "${PG_CPUS:-2}", memory: "${PG_MEMORY:-1g}"}
|
|
|
|
valkey:
|
|
image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a
|
|
user: "999:1000"
|
|
command:
|
|
- valkey-server
|
|
- /run/secrets/valkey.conf
|
|
- --appendonly
|
|
- "yes"
|
|
- --appendfsync
|
|
- always
|
|
- --maxmemory
|
|
- 64mb
|
|
- --maxmemory-policy
|
|
- volatile-lru
|
|
secrets: [valkey.conf]
|
|
volumes: [valkeydata:/data]
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli ping | grep -q PONG"]
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 30
|
|
stop_grace_period: 10s
|
|
restart: unless-stopped
|
|
networks: [backend]
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "0.5", memory: 128m}
|
|
|
|
s3:
|
|
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
|
|
environment:
|
|
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user
|
|
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password
|
|
RUSTFS_ADDRESS: ":9000"
|
|
RUSTFS_CONSOLE_ENABLE: "false"
|
|
RUSTFS_REGION: ${S3_REGION:-us-east-1}
|
|
RUSTFS_OBS_LOG_DIRECTORY: ""
|
|
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
|
|
secrets: [s3_root_user, s3_root_password]
|
|
volumes: [s3data:/data]
|
|
read_only: true
|
|
tmpfs: ["/tmp:size=64m"]
|
|
cap_drop: [ALL]
|
|
security_opt: ["no-new-privileges:true"]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"]
|
|
interval: 2s
|
|
timeout: 3s
|
|
retries: 30
|
|
start_period: 20s
|
|
stop_grace_period: 10s
|
|
restart: unless-stopped
|
|
networks: [backend]
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
|
|
|
|
s3-init:
|
|
<<: *s3-client
|
|
entrypoint: ["/bin/sh", "/s3/init.sh"]
|
|
depends_on:
|
|
s3:
|
|
condition: service_healthy
|
|
|
|
s3-cli:
|
|
<<: *s3-client
|
|
profiles: [tools]
|
|
entrypoint: ["/bin/sh", "/s3/cli.sh"]
|
|
|
|
k6:
|
|
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
|
|
profiles: [bench]
|
|
environment:
|
|
BASE_URL: http://lb:8080
|
|
volumes: [./infra/k6:/scripts:ro]
|
|
networks: [edge]
|
|
deploy:
|
|
resources:
|
|
limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"}
|
|
|
|
secrets:
|
|
postgres_password:
|
|
file: ./secrets/postgres_password
|
|
database_url:
|
|
file: ./secrets/database_url
|
|
valkey.conf:
|
|
file: ./secrets/valkey.conf
|
|
cache_url:
|
|
file: ./secrets/cache_url
|
|
cursor_secret:
|
|
file: ./secrets/cursor_secret
|
|
s3_root_user:
|
|
file: ./secrets/s3_root_user
|
|
s3_root_password:
|
|
file: ./secrets/s3_root_password
|
|
s3_access_key_id:
|
|
file: ./secrets/s3_access_key_id
|
|
s3_secret_access_key:
|
|
file: ./secrets/s3_secret_access_key
|
|
|
|
volumes:
|
|
pgdata:
|
|
valkeydata:
|
|
s3data:
|
|
|
|
networks:
|
|
edge:
|
|
backend:
|
|
internal: true
|