diff --git a/Makefile b/Makefile index ca4d7b27..8e22650b 100644 --- a/Makefile +++ b/Makefile @@ -10,14 +10,17 @@ COMPOSE ?= docker compose RATE ?= 100 API_IMAGE ?= mtusi-messenger:lab3 FROM_REV ?= -SCRIPTS_PY = scripts/probe.py scripts/limits.py +SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/scaling.py +TRACK1 = evidence/track1 +SCALING = $(PYTHON) scripts/scaling.py --fresh-db --start-per-cpu 150 PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ persistence-evidence storage-evidence cache-evidence test-faults seed explain \ - limits detection failover-each rolling-update level4 + limits detection failover-each rolling-update level4 \ + track1 track1-host track1-a track1-b help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -25,6 +28,7 @@ help: @echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes" @echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа" @echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps" + @echo "track1 = track1-host, track1-a (1x4/2x2/4x1), track1-b (1x1/2x1/8x1) → $(TRACK1)" check: $(PYTHON) scripts/check_contract.py @@ -114,6 +118,17 @@ endif level4: limits lb-check detection failover-each rolling-update +track1: track1-host track1-a track1-b + +track1-host: + sh scripts/host-snapshot.sh $(TRACK1)/host.txt + +track1-a: + $(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 + +track1-b: + $(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 + failover-evidence: sh scripts/failover-evidence.sh "$(BASE_URL)" diff --git a/openspec/changes/add-lab3-scaling-cost/tasks.md b/openspec/changes/add-lab3-scaling-cost/tasks.md index b07ce578..3a04fead 100644 --- a/openspec/changes/add-lab3-scaling-cost/tasks.md +++ b/openspec/changes/add-lab3-scaling-cost/tasks.md @@ -5,8 +5,8 @@ ## 2. Сэмплеры и оркестратор -- [ ] 2.1 `scripts/docker-stats.sh`, `scripts/lb-stats.sh`, `scripts/pg-stats.sh` (activity, statements, reset, размер базы) -- [ ] 2.2 `scripts/scaling.py`: конфигурация `RxC` с воркерами, пулом, очередью LB на реплику и закреплением CPU; поиск колена с повторами; `summary.csv` и сырьё шага; `probe.py` — ожидание реплик в LB +- [x] 2.1 `scripts/docker-stats.sh`, `scripts/lb-stats.sh`, `scripts/pg-stats.sh` (activity, statements, reset, размер базы) +- [x] 2.2 `scripts/scaling.py`: конфигурация `RxC` с воркерами, пулом, очередью LB на реплику и закреплением CPU; поиск колена с повторами; `summary.csv` и сырьё шага; `probe.py` — ожидание реплик в LB - [ ] 2.3 `tools/plot_track1.py` и `make plots`; цели `make track1-a`, `track1-b`, `track1-compare`, `track1` - [ ] 2.4 Проверка группы: `make lint`, `make check`, пробный шаг `scaling.py` на одной интенсивности diff --git a/pyproject.toml b/pyproject.toml index b7e26dc3..ff07cf81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"] -"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"] +"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "SIM905"] "src/messenger/*/infrastructure/**" = ["TID251", "ARG002"] "src/messenger/*/domain/errors.py" = ["N818"] "src/messenger/*/application/errors.py" = ["N818"] diff --git a/scripts/docker-stats.sh b/scripts/docker-stats.sh new file mode 100755 index 00000000..f3c524cd --- /dev/null +++ b/scripts/docker-stats.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +out=${1:?usage: docker-stats.sh } +cd "$(dirname "$0")/.." +project=$(docker compose config | sed -n 's/^name: //p') +echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out" +while :; do + ids=$(docker ps -q --filter "label=com.docker.compose.project=$project") + [ -z "$ids" ] || docker stats --no-stream --format "$(date +%s),{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true + sleep "${INTERVAL:-1}" +done diff --git a/scripts/host-snapshot.sh b/scripts/host-snapshot.sh new file mode 100755 index 00000000..5fc11676 --- /dev/null +++ b/scripts/host-snapshot.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +out=${1:?usage: host-snapshot.sh } +cd "$(dirname "$0")/.." +mkdir -p "$(dirname "$out")" +{ + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null || nproc) CPU, macOS $(sw_vers -productVersion 2>/dev/null || uname -r)" + echo "docker: $(docker version --format '{{.Server.Version}}'), compose $(docker compose version --short), VM $(docker info --format '{{.NCPU}} CPU, {{.MemTotal}} bytes')" + echo + echo "\$ docker stats --no-stream --all # every container on the host, before the series" + docker stats --no-stream --all --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.PIDs}}' + echo + echo "\$ docker ps --format '{{.Names}} {{.Image}} {{.Status}}'" + docker ps --format '{{.Names}} {{.Image}} {{.Status}}' + echo + echo "\$ docker compose --profile bench config --images" + docker compose --profile bench config --images +} > "$out" +cat "$out" diff --git a/scripts/lb-stats.sh b/scripts/lb-stats.sh new file mode 100755 index 00000000..13754a59 --- /dev/null +++ b/scripts/lb-stats.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +out=${1:?usage: lb-stats.sh } +cd "$(dirname "$0")/.." +url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv} +curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out" +while :; do + ts=$(date +%s) + curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true + sleep "${INTERVAL:-1}" +done diff --git a/scripts/pg-stats.sh b/scripts/pg-stats.sh new file mode 100755 index 00000000..736a5d5a --- /dev/null +++ b/scripts/pg-stats.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +mode=${1:?usage: pg-stats.sh activity | statements | size | reset} +cd "$(dirname "$0")/.." +psql() { + docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@" +} +case $mode in +reset) + psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null + ;; +size) + psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)" + ;; +statements) + psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}" + ;; +activity) + echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}" + while :; do + ts=$(date +%s) + psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true + sleep "${INTERVAL:-1}" + done + ;; +*) + echo "unknown mode $mode" >&2 + exit 2 + ;; +esac diff --git a/scripts/scaling.py b/scripts/scaling.py new file mode 100755 index 00000000..a33846f7 --- /dev/null +++ b/scripts/scaling.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +import argparse +import collections +import csv +import json +import math +import os +import re +import signal +import statistics +import subprocess +import sys +import time +from pathlib import Path + +from probe import ROOT, k6_metrics, lb_up, run_k6 + +SCRIPTS = ROOT / "scripts" +UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9} +PINNED = { + "PG_CPUSET": "0-1", + "LB_CPUSET": "2", + "VALKEY_CPUSET": "3", + "K6_CPUSET": "4-6", + "K6_CPUS": "3", + "K6_MEMORY": "1g", +} +API_FIRST_CPU = 7 +FIELDS = ( + "variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat " + "started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio " + "dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres " + "cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max " + "pg_active_avg pg_active_max pg_idle_avg pg_idle_tx_avg pg_lock_avg pg_lock_max pg_lwlock_avg " + "pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms" +).split() + + +def compose(*args, env=None): + command = ["docker", "compose", *args] + return subprocess.run( + command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True + ).stdout + + +def size(text): + match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip()) + return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0 + + +def rows(path, start, end): + with path.open() as handle: + return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end] + + +def mean(values): + return round(statistics.mean(values), 2) if values else 0.0 + + +def docker_usage(path, project, start, end): + ticks = collections.defaultdict(collections.Counter) + memory = collections.Counter() + for r in rows(path, start, end): + name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :]) + ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0) + memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20) + cpu = { + name: mean([t[name] for t in ticks.values() if name in t]) + for name in ("api", "postgres", "lb", "valkey", "k6") + } + usage = {f"cpu_{name}": value for name, value in cpu.items()} + usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")}) + return usage + + +def pg_usage(path, start, end): + ticks = collections.defaultdict(collections.Counter) + for r in rows(path, start, end): + count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"] + tick = ticks[r["ts"]] + tick["backends"] += count + tick["active"] += count * (state == "active") + tick["idle"] += count * (state == "idle") + tick["idle_tx"] += count * state.startswith("idle in transaction") + tick["lock"] += count * (wait == "Lock") + tick["lwlock"] += count * (wait == "LWLock") + tick["io"] += count * (wait == "IO") + series = list(ticks.values()) + return { + "pg_backends_max": max((t["backends"] for t in series), default=0), + "pg_active_avg": mean([t["active"] for t in series]), + "pg_active_max": max((t["active"] for t in series), default=0), + "pg_idle_avg": mean([t["idle"] for t in series]), + "pg_idle_tx_avg": mean([t["idle_tx"] for t in series]), + "pg_lock_avg": mean([t["lock"] for t in series]), + "pg_lock_max": max((t["lock"] for t in series), default=0), + "pg_lwlock_avg": mean([t["lwlock"] for t in series]), + "pg_io_avg": mean([t["io"] for t in series]), + } + + +def lb_usage(path, start, end): + backend = [ + r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND" + ] + + def numbers(field): + return [int(r[field] or 0) for r in backend] + + return { + "lb_scur_max": max(numbers("scur"), default=0), + "lb_qcur_max": max(numbers("qcur"), default=0), + "lb_qtime_max_ms": max(numbers("qtime"), default=0), + "lb_rtime_avg_ms": mean(numbers("rtime")), + } + + +def trend(path, name): + metrics = json.loads(Path(path).read_text())["metrics"] + found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {}) + return round(found.get("values", {}).get("p(95)", 0.0), 2) + + +def sampler(*command): + return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True) + + +def stop(process): + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + process.wait() + + +def wait_lb(expected, timeout=120): + deadline, stable = time.monotonic() + timeout, 0 + while stable < 4: + if time.monotonic() > deadline: + raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})") + stable = stable + 1 if lb_up() == expected else 0 + time.sleep(0.5) + + +def layout(config, args, variant): + name, _, extra = variant.partition(":") + replicas, cpus = (float(v) for v in config.split("x")) + replicas = int(replicas) + workers = args.workers or max(1, int(cpus)) + width = max(1, math.ceil(replicas * cpus)) + cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "") + memory = args.api_memory or args.api_memory_total // replicas + settings = { + "API_REPLICAS": str(replicas), + "API_CPUS": f"{cpus:g}", + "API_MEMORY": f"{memory}m", + "WEB_CONCURRENCY": str(workers), + "DB_POOL_MAX_SIZE": str(args.pool), + "API_SERVER_MAXCONN": str(args.conn_per_worker * workers), + "API_CPUSET": cpuset if args.pin else "", + } + settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))])) + base = { + "variant": name, + "config": config, + "replicas": replicas, + "cpus": cpus, + "memory_mb": memory, + "workers": int(settings["WEB_CONCURRENCY"]), + "pool": int(settings["DB_POOL_MAX_SIZE"]), + "maxconn": int(settings["API_SERVER_MAXCONN"]), + "api_cpuset": settings["API_CPUSET"], + } + return base, settings + + +def apply(base, settings, env): + env = dict(env, **settings) + compose("up", "-d", "--wait", "--wait-timeout", "180", env=env) + wait_lb(base["replicas"]) + return env + + +def measure(folder, args, rate, env): + subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True) + database = subprocess.run( + [str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True + ).stdout.split(",") + samplers = [ + sampler("docker-stats.sh", str(folder / "docker_stats.csv")), + sampler("lb-stats.sh", str(folder / "lb_stats.csv")), + sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")), + ] + load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration)) + try: + code, started = run_k6(folder / "k6", load) + finally: + for process in samplers: + stop(process) + subprocess.run( + [str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True + ) + if started is None: + raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}") + return code, started + args.warmup, database + + +def step(base, rate, repeat, args, env): + folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}" + folder.mkdir(parents=True, exist_ok=True) + code, start, database = measure(folder, args, rate, env) + end = start + args.duration + summary = folder / "k6/summary.json" + metrics = k6_metrics(summary, args.duration) + saturated = ( + metrics["failed_ratio"] > args.max_error_ratio + or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration + or metrics["p95_ms"] > args.slo_p95_ms + ) + row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics) + row["started"] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(start)) + row["db_mb"] = round(int(database[0]) / 2**20, 1) + row["messages"] = int(database[1]) + row["p95_post_ms"] = trend(summary, "post_message") + row["p95_get_ms"] = trend(summary, "get_history") + row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end)) + row.update(pg_usage(folder / "pg_activity.csv", start, end)) + row.update(lb_usage(folder / "lb_stats.csv", start, end)) + print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True) + return row + + +def rates(args, base): + if args.rates: + return [int(r) for r in args.rates.split(",")] + factor, top = (float(v) for v in args.search.split(":")) + value, values = args.start_per_cpu * base["replicas"] * base["cpus"], [] + while value <= top: + values.append(int(round(value, -1))) + value *= factor + return values + + +def arguments(): + parser = argparse.ArgumentParser(description="Track 1: replicas against throughput") + parser.add_argument("--out", required=True) + parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica") + parser.add_argument( + "--variant", + dest="variants", + action="append", + help="NAME[:KEY=VALUE,...]; several variants alternate per repeat", + ) + parser.add_argument("--fresh-db", action="store_true", help="drop volumes before a config") + parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas") + parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica") + parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus") + parser.add_argument("--pool", type=int, default=8, help="DB pool per worker") + parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker") + parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE") + parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs") + parser.add_argument("--rates", help="comma separated offered rates, rps") + parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu") + parser.add_argument("--start-per-cpu", type=float, default=100.0) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--warmup", type=int, default=10) + parser.add_argument("--duration", type=int, default=30) + parser.add_argument("--pause", type=float, default=5.0) + parser.add_argument("--slo-p95-ms", type=float, default=500.0) + parser.add_argument("--max-error-ratio", type=float, default=0.01) + parser.add_argument("--keep", action="store_true", help="leave the last layout running") + args = parser.parse_args() + args.variants = args.variants or ["base"] + if len(args.variants) > 1 and not args.rates: + parser.error("several variants need explicit --rates") + return args + + +def block(args, output, config, variant, repeats): + base, settings = layout(config, args, variant) + current, streak = apply(base, settings, args.base_env), 0 + for rate in rates(args, base): + results = [] + for repeat in repeats: + results.append(step(base, rate, repeat, args, current)) + time.sleep(args.pause) + output.writerows(results) + streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0 + if streak == 2: + break + + +def series(args, output): + every = range(1, args.repeats + 1) + for config in args.configs.split(","): + if args.fresh_db: + compose("down", "-v", env=args.base_env) + if len(args.variants) == 1: + block(args, output, config, args.variants[0], every) + continue + for repeat in every: + for variant in args.variants: + block(args, output, config, variant, [repeat]) + + +def main(): + args = arguments() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + args.project = json.loads(compose("config", "--format", "json"))["name"] + args.base_env = dict(os.environ, **(PINNED if args.pin else {})) + summary = out / "summary.csv" + fresh = not summary.exists() + try: + with summary.open("a", newline="", buffering=1) as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore") + if fresh: + writer.writeheader() + series(args, writer) + finally: + if not args.keep: + compose("up", "-d", "--wait", "--wait-timeout", "180") + return 0 + + +if __name__ == "__main__": + sys.exit(main())