Files
mtusi2026lab4/compose.yaml
T
ArtyomandClaude Opus 5.5 a774a2fbbd Measure upload memory and bound the buffers along the chain
upload_memory.py recreates the replicas and the balancer for every run
of 10, 20 and 40 concurrent uploads of 1 and 10 MiB with 64 slots
(effectively no limit) and with 3, drives them through lab4_seed.py
from the prepared tooling, and keeps per-run JSON plus a summary with
the cgroup memory.peak of each replica and of the balancer and the
oom_kill counters. The seed script now also records memory.current
before the load and reads X-Instance-Id regardless of case.

The first run showed that slots alone only move a waiting body out of
the worker: with 3 slots and 40 uploads of 10 MiB the kernel grew the
socket buffers of HAProxy to about 110 MB and the 128 MiB balancer was
killed. HAProxy now caps its client and server socket buffers at 64
KiB and the API replicas cap tcp_rmem and tcp_wmem at 256 KiB, which
keeps the balancer at 30 MiB. glibc raised its mmap threshold after
the first large free and kept freed 10 MiB bodies in the heap, so the
API fixes MALLOC_MMAP_THRESHOLD_ at 128 KiB.

With these settings 40 concurrent uploads of 10 MiB peak at 323 MiB of
512 per replica and all 80 succeed, while without the slot limit the
same load reaches 512 MiB, a worker is killed and only 31 of 80
succeed. UPLOAD_CONCURRENCY stays at 3.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-29 00:14:09 +03:00

277 lines
8.3 KiB
YAML

x-app: &app
image: ${API_IMAGE:-mtusi-messenger:lab4}
networks: [backend]
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-3}
UPLOAD_GRACE_SECONDS: ${UPLOAD_GRACE_SECONDS:-900}
UPLOAD_SWEEP_INTERVAL_SECONDS: ${UPLOAD_SWEEP_INTERVAL_SECONDS:-30}
UPLOAD_SWEEP_BATCH: ${UPLOAD_SWEEP_BATCH:-100}
x-s3-client: &s3-client
image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699
read_only: true
tmpfs: ["/tmp:size=16m"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
volumes: ["./infra/s3:/s3:ro"]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
networks: [backend]
deploy:
resources:
limits: {cpus: "0.5", memory: 64m}
services:
lb:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
environment:
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports:
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
networks: [edge, backend]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
sysctls:
net.ipv4.ip_local_port_range: "10240 65535"
net.ipv4.tcp_tw_reuse: "1"
stop_signal: SIGUSR1
stop_grace_period: 8s
restart: unless-stopped
read_only: true
cap_drop: [ALL]
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1}", memory: 128m}
api:
<<: *app
build: .
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
MALLOC_MMAP_THRESHOLD_: ${MALLOC_MMAP_THRESHOLD_:-131072}
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
healthcheck:
test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
stop_grace_period: 8s
restart: unless-stopped
read_only: true
tmpfs: [/tmp]
sysctls:
net.ipv4.tcp_rmem: "4096 65536 262144"
net.ipv4.tcp_wmem: "4096 65536 262144"
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_started
s3-init:
condition: service_completed_successfully
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-512m}"}
migrate:
<<: *app
pull_policy: never
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url]
healthcheck:
disable: true
restart: "no"
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
postgres:
image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b
user: "70:70"
command:
- postgres
- -c
- max_connections=${PG_MAX_CONNECTIONS:-200}
- -c
- shared_buffers=${PG_SHARED_BUFFERS:-256MB}
- -c
- shared_preload_libraries=pg_stat_statements
- -c
- pg_stat_statements.track=top
- -c
- track_io_timing=on
- -c
- idle_in_transaction_session_timeout=10s
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [pgdata:/var/lib/postgresql/data]
shm_size: 256m
healthcheck:
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2}", memory: "${PG_MEMORY:-1g}"}
valkey:
image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a
user: "999:1000"
command:
- valkey-server
- /run/secrets/valkey.conf
- --appendonly
- "yes"
- --appendfsync
- always
- --maxmemory
- 64mb
- --maxmemory-policy
- volatile-lru
secrets: [valkey.conf]
volumes: [valkeydata:/data]
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/valkey.conf)\" valkey-cli ping | grep -q PONG"]
interval: 2s
timeout: 3s
retries: 30
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "0.5", memory: 128m}
s3:
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
environment:
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_REGION: ${S3_REGION:-us-east-1}
RUSTFS_OBS_LOG_DIRECTORY: ""
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
secrets: [s3_root_user, s3_root_password]
volumes: [s3data:/data]
read_only: true
tmpfs: ["/tmp:size=64m"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"]
interval: 2s
timeout: 3s
retries: 30
start_period: 20s
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
s3-init:
<<: *s3-client
entrypoint: ["/bin/sh", "/s3/init.sh"]
depends_on:
s3:
condition: service_healthy
s3-cli:
<<: *s3-client
profiles: [tools]
entrypoint: ["/bin/sh", "/s3/cli.sh"]
k6:
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
profiles: [bench]
environment:
BASE_URL: http://lb:8080
volumes: [./infra/k6:/scripts:ro]
networks: [edge]
deploy:
resources:
limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"}
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
valkey.conf:
file: ./secrets/valkey.conf
cache_url:
file: ./secrets/cache_url
cursor_secret:
file: ./secrets/cursor_secret
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata:
valkeydata:
s3data:
networks:
edge:
backend:
internal: true