forked from mtusi2026/mtusi2026lab4
upload_memory.py recreates the replicas and the balancer for every run of 10, 20 and 40 concurrent uploads of 1 and 10 MiB with 64 slots (effectively no limit) and with 3, drives them through lab4_seed.py from the prepared tooling, and keeps per-run JSON plus a summary with the cgroup memory.peak of each replica and of the balancer and the oom_kill counters. The seed script now also records memory.current before the load and reads X-Instance-Id regardless of case. The first run showed that slots alone only move a waiting body out of the worker: with 3 slots and 40 uploads of 10 MiB the kernel grew the socket buffers of HAProxy to about 110 MB and the 128 MiB balancer was killed. HAProxy now caps its client and server socket buffers at 64 KiB and the API replicas cap tcp_rmem and tcp_wmem at 256 KiB, which keeps the balancer at 30 MiB. glibc raised its mmap threshold after the first large free and kept freed 10 MiB bodies in the heap, so the API fixes MALLOC_MMAP_THRESHOLD_ at 128 KiB. With these settings 40 concurrent uploads of 10 MiB peak at 323 MiB of 512 per replica and all 80 succeed, while without the slot limit the same load reaches 512 MiB, a worker is killed and only 31 of 80 succeed. UPLOAD_CONCURRENCY stays at 3. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
169 lines
5.6 KiB
Makefile
169 lines
5.6 KiB
Makefile
PYTHON ?= python3
|
|
BASE_URL ?= http://localhost:8080
|
|
CA_FILE ?=
|
|
ACTION_SCRIPT ?=
|
|
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
|
|
|
VENV ?= .venv
|
|
UV ?= uv
|
|
COMPOSE ?= docker compose
|
|
RATE ?= 100
|
|
API_IMAGE ?= mtusi-messenger:lab4
|
|
FROM_REV ?=
|
|
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
|
scripts/upload_memory.py
|
|
SCENARIO ?= failover-upload
|
|
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
|
|
|
.PHONY: help check validate test load test-persistence test-failover \
|
|
install lint format typecheck student-test \
|
|
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
|
|
persistence-evidence storage-evidence cache-evidence test-faults seed explain \
|
|
limits detection failover-each rolling-update level4 \
|
|
scenario s3-check attachments-evidence upload-faults upload-memory
|
|
|
|
help:
|
|
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
|
@echo "install/lint/typecheck/student-test: студенческий тулчейн"
|
|
@echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes"
|
|
@echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа"
|
|
@echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps"
|
|
@echo "scenario SCENARIO=failover-upload|s3-outage → evidence/<сценарий>.json; s3-check; attachments-evidence"
|
|
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
|
|
@echo "upload-memory: пик памяти реплик при 10/20/40 одновременных загрузках 1 и 10 MiB → evidence/upload-memory/"
|
|
|
|
check:
|
|
$(PYTHON) scripts/check_contract.py
|
|
|
|
validate:
|
|
$(PYTHON) scripts/validate_openapi.py
|
|
|
|
test:
|
|
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
|
|
|
load:
|
|
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
|
|
|
test-persistence:
|
|
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
|
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
|
|
|
test-failover:
|
|
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
|
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
|
|
|
install:
|
|
$(UV) sync --group dev
|
|
|
|
lint:
|
|
$(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY)
|
|
$(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY)
|
|
$(VENV)/bin/ty check src student-tests
|
|
|
|
format:
|
|
$(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY)
|
|
$(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY)
|
|
|
|
typecheck:
|
|
$(VENV)/bin/ty check src student-tests
|
|
|
|
student-test:
|
|
$(VENV)/bin/pytest
|
|
|
|
secrets:
|
|
sh scripts/secrets.sh
|
|
|
|
up: secrets
|
|
$(COMPOSE) up -d --build --wait
|
|
|
|
down:
|
|
$(COMPOSE) down
|
|
|
|
reset-data:
|
|
@echo "Удаляются volumes с учебными данными стенда"
|
|
$(COMPOSE) down -v --remove-orphans
|
|
|
|
migrate:
|
|
$(COMPOSE) run --rm migrate
|
|
|
|
ps:
|
|
$(COMPOSE) ps
|
|
|
|
logs:
|
|
$(COMPOSE) logs --tail 200
|
|
|
|
restore:
|
|
scripts/restore.sh
|
|
|
|
lb-check:
|
|
$(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json
|
|
|
|
limits:
|
|
$(PYTHON) scripts/limits.py --out evidence/limits.txt
|
|
|
|
detection:
|
|
sh scripts/detection.sh evidence/detection
|
|
|
|
failover-each:
|
|
RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each
|
|
|
|
rolling-update:
|
|
ifneq ($(FROM_REV),)
|
|
git archive "$(FROM_REV)" | docker build -q -t "$(API_IMAGE)" -
|
|
$(COMPOSE) up -d --no-deps --force-recreate --wait api
|
|
scripts/lb-wait.sh
|
|
endif
|
|
$(COMPOSE) build -q api
|
|
RATE=$(RATE) $(PYTHON) scripts/probe.py load --expect replace --duration 90 \
|
|
--out evidence/rolling$(if $(FROM_REV),-from-$(FROM_REV)) \
|
|
-- scripts/rolling-update.sh
|
|
|
|
level4: limits lb-check detection failover-each rolling-update
|
|
|
|
scenario:
|
|
$(PYTHON) scripts/lab4_scenarios.py $(SCENARIO) --base-url "$(BASE_URL)" --restore --output evidence/$(SCENARIO).json
|
|
scripts/lb-wait.sh
|
|
|
|
s3-check:
|
|
sh scripts/s3-check.sh > evidence/s3-security.txt; status=$$?; cat evidence/s3-security.txt; exit $$status
|
|
|
|
attachments-evidence:
|
|
sh scripts/evidence-attachments.sh
|
|
|
|
upload-faults:
|
|
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s --no-header --tb=short -p no:cacheprovider \
|
|
student-tests/stack/test_attachment_faults.py > evidence/upload-faults.txt; \
|
|
status=$$?; cat evidence/upload-faults.txt; exit $$status
|
|
|
|
upload-memory:
|
|
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)"
|
|
|
|
failover-evidence:
|
|
sh scripts/failover-evidence.sh "$(BASE_URL)"
|
|
|
|
stack-evidence:
|
|
sh scripts/evidence-stack.sh
|
|
|
|
persistence-evidence:
|
|
sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt
|
|
cat evidence/persistence.txt
|
|
|
|
storage-evidence:
|
|
sh scripts/evidence-storage.sh
|
|
|
|
cache-evidence:
|
|
sh scripts/evidence-cache.sh > evidence/cache.txt
|
|
cat evidence/cache.txt
|
|
|
|
test-faults:
|
|
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
|
|
|
|
seed:
|
|
$(PSQL) -f - < scripts/sql/seed-history.sql
|
|
|
|
explain: seed
|
|
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \
|
|
echo "\$$ make explain # scripts/sql/explain-history.sql"; \
|
|
$(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt
|
|
cat evidence/explain-history.txt
|