From 05d69ba6ac13a373326ce03504a94ae89ec69687 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 5 Nov 2026 22:42:51 +0300 Subject: [PATCH] Preserve scaling data and reject contaminated benchmark attempts --- Makefile | 31 ++++--- .../changes/remeasure-lab3-scaling/tasks.md | 4 +- scripts/scaling.py | 82 +++++++++++++++++-- student-tests/unit/test_scaling.py | 54 ++++++++++++ tools/plot_track1.py | 9 +- 5 files changed, 157 insertions(+), 23 deletions(-) create mode 100644 student-tests/unit/test_scaling.py diff --git a/Makefile b/Makefile index 4a541ebe..8f9112a6 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,10 @@ RATE ?= 100 FROM_REV ?= SHIFT ?= 0 SCRIPTS_PY = alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools -TRACK1 = evidence/track1 -SCALING = $(PYTHON) scripts/scaling.py --fresh-db -BEFORE_REV ?= edc091d +TRACK1 ?= evidence/track1-sqlalchemy +SCALING = $(PYTHON) scripts/scaling.py +BEFORE_REV ?= 5c8f605cd3bd40245109348463d3e7111379b2d3 +BASELINE = --env API_IMAGE=mtusi-messenger:$(BEFORE_REV) PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q PYTEST_ARGS ?= @@ -22,7 +23,9 @@ PYTEST_ARGS ?= secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \ limits detection failover-each rolling-update level4 \ - track1 track1-host track1-a track1-b track1-compare track1-control plots + track1 track1-images track1-host track1-a track1-b track1-compare track1-control plots + +.NOTPARALLEL: help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -128,30 +131,32 @@ endif level4: limits lb-check detection failover-each rolling-update -track1: track1-host track1-a track1-b track1-compare track1-control plots +track1: track1-images track1-host track1-a track1-b track1-compare track1-control plots + +track1-images: + git archive "$(BEFORE_REV)" | docker build -q -t mtusi-messenger:$(BEFORE_REV) - + $(COMPOSE) build -q api track1-host: sh scripts/host-snapshot.sh $(TRACK1)/host.txt track1-a: $(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 \ - --start-per-cpu 150 + --start-per-cpu 150 $(BASELINE) track1-b: $(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 \ - --start-per-cpu 450 --search 1.5:9000 + --start-per-cpu 300 --search 1.5:9000 $(BASELINE) -track1-compare: - git archive "$(BEFORE_REV)" | docker build -q -t mtusi-messenger:$(BEFORE_REV) - - $(COMPOSE) build -q api - $(SCALING) --out $(TRACK1)/compare --configs 8x1 --api-memory 512 --rates 3600,5400,8100 \ +track1-compare: track1-images + $(SCALING) --out $(TRACK1)/compare --configs 8x1 --api-memory 512 --rates 2400,3600,5400 \ --variant before:API_IMAGE=mtusi-messenger:$(BEFORE_REV) --variant after track1-control: - $(SCALING) --out $(TRACK1)/control-pool --configs 8x1 --api-memory 512 --rates 5400 \ + $(SCALING) $(BASELINE) --out $(TRACK1)/control-pool --configs 8x1 --api-memory 512 --rates 5400 \ --variant pool4:DB_POOL_MAX_SIZE=4 --variant pool8 --variant pool16:DB_POOL_MAX_SIZE=16 \ --variant maxconn16:API_SERVER_MAXCONN=16 --variant maxconn64:API_SERVER_MAXCONN=64 - $(SCALING) --out $(TRACK1)/control-cpus --configs 4x1,8x0.5 --api-memory 512 --rates 4560,6830 \ + $(SCALING) $(BASELINE) --out $(TRACK1)/control-cpus --configs 4x1,8x0.5 --api-memory-total 2048 --rates 3040,4560 \ --variant pinned plots: diff --git a/openspec/changes/remeasure-lab3-scaling/tasks.md b/openspec/changes/remeasure-lab3-scaling/tasks.md index 3479d42d..2716c00b 100644 --- a/openspec/changes/remeasure-lab3-scaling/tasks.md +++ b/openspec/changes/remeasure-lab3-scaling/tasks.md @@ -1,7 +1,7 @@ ## 1. Методика и безопасная обвязка -- [ ] 1.1 Проверить пилот, исключить удаление volumes и перезапись попыток, сохранять provenance и реальные лимиты. -- [ ] 1.2 Проверить обвязку собственными тестами, make check/lint/student-test и smoke. +- [x] 1.1 Проверить пилот, исключить удаление volumes и перезапись попыток, сохранять provenance и реальные лимиты. +- [x] 1.2 Проверить обвязку собственными тестами, make check/lint/student-test и smoke. ## 2. Базовые серии SQLAlchemy diff --git a/scripts/scaling.py b/scripts/scaling.py index 657102d1..793c8c7c 100755 --- a/scripts/scaling.py +++ b/scripts/scaling.py @@ -28,7 +28,7 @@ PINNED = { } API_FIRST_CPU = 7 FIELDS = ( - "variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat " + "raw_dir variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat " "started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio " "dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres " "cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max " @@ -228,6 +228,58 @@ def apply(base, settings, env, out): return env +def deployment(env): + ids = compose("ps", "-q", env=env).split() + template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}" + result = subprocess.run( + ["docker", "inspect", "--format", template, *ids], + check=True, + capture_output=True, + text=True, + ) + containers = [] + for line in result.stdout.splitlines(): + name, image, config = line.split(" ", 2) + limits = json.loads(config) + containers.append( + dict( + name=json.loads(name), + image=json.loads(image), + **{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")}, + ) + ) + return containers + + +def provenance(folder, args, env): + stamp = time.strftime("%FT%TZ", time.gmtime()) + metadata = { + "date": stamp, + "commit": args.commit, + "project": args.project, + "command": sys.argv, + "api_image": env.get("API_IMAGE", "compose default"), + "containers": args.deployment, + } + (folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n") + for path in folder.rglob("*"): + if path.suffix in {".csv", ".txt"}: + path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text()) + elif path.name == "summary.json": + data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text())) + path.write_text(json.dumps(data) + "\n") + + +def attempt_folder(out, base, rate, repeat): + name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}" + attempt = 1 + while (Path(out) / f"{name}_attempt{attempt}").exists(): + attempt += 1 + folder = Path(out) / f"{name}_attempt{attempt}" + folder.mkdir(parents=True) + return folder + + def measure(folder, args, rate, env): subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True) database = subprocess.run( @@ -255,8 +307,7 @@ def measure(folder, args, rate, env): def step(base, rate, repeat, args, env): - folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}" - folder.mkdir(parents=True, exist_ok=True) + folder = attempt_folder(args.out, base, rate, repeat) code, start, database, wall = measure(folder, args, rate, env) end = start + args.duration summary = folder / "k6/summary.json" @@ -266,7 +317,15 @@ def step(base, rate, repeat, args, env): or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration or metrics["p95_ms"] > args.slo_p95_ms ) - row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics) + row = dict( + base, + raw_dir=folder.name, + rate=rate, + repeat=repeat, + k6_exit=code, + saturated=saturated, + **metrics, + ) row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start)) row["db_mb"] = round(int(database[0]) / 2**20, 1) row["messages"] = int(database[1]) @@ -284,7 +343,13 @@ def step(base, rate, repeat, args, env): row["cpu_other"] > args.max_other or row["gap_s"] > args.max_gap or wall > args.warmup + args.duration + args.max_overhead + or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2")) + or not rows(folder / "pg_activity.csv", start, end) + or not rows(folder / "lb_stats.csv", start, end) ) + provenance(folder, args, env) + if code or not metrics["requests"]: + raise SystemExit(f"invalid k6 run: {folder}") print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True) return row @@ -310,7 +375,6 @@ def arguments(): action="append", help="NAME[:KEY=VALUE,...]; several variants alternate per repeat", ) - parser.add_argument("--fresh-db", action="store_true", help="drop volumes before a config") parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas") parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica") parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus") @@ -344,6 +408,7 @@ def arguments(): def block(args, output, config, variant, repeats): base, settings = layout(config, args, variant) current, streak = apply(base, settings, args.base_env, args.out), 0 + args.deployment = deployment(current) for rate in rates(args, base): results = [] for repeat in repeats: @@ -354,6 +419,8 @@ def block(args, output, config, variant, repeats): break args.rejected.writerow(row) wait_quiet(args) + else: + raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}") results.append(row) output.writerows(results) streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0 @@ -364,8 +431,6 @@ def block(args, output, config, variant, repeats): def series(args, output): every = range(args.first_repeat, args.first_repeat + args.repeats) for config in args.configs.split(","): - if args.fresh_db: - compose("down", "-v", env=args.base_env) if len(args.variants) == 1: block(args, output, config, args.variants[0], every) continue @@ -380,6 +445,8 @@ def table(path): with path.open("a", newline="", buffering=1) as handle: writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore") if fresh: + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n") writer.writeheader() yield writer @@ -389,6 +456,7 @@ def main(): out = Path(args.out) out.mkdir(parents=True, exist_ok=True) args.project = json.loads(compose("config", "--format", "json"))["name"] + args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() args.base_env = dict(os.environ, **(PINNED if args.pin else {})) try: with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected: diff --git a/student-tests/unit/test_scaling.py b/student-tests/unit/test_scaling.py new file mode 100644 index 00000000..4f7cfff3 --- /dev/null +++ b/student-tests/unit/test_scaling.py @@ -0,0 +1,54 @@ +import importlib.util +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def scaling(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + scripts = Path(__file__).resolve().parents[2] / "scripts" + monkeypatch.syspath_prepend(str(scripts)) + spec = importlib.util.spec_from_file_location("scaling", scripts / "scaling.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_retry_preserves_previous_raw_data(scaling: ModuleType, tmp_path: Path) -> None: + base = {"variant": "base", "config": "1x1"} + first = scaling.attempt_folder(tmp_path, base, 100, 1) + (first / "k6.txt").write_text("failed attempt") + second = scaling.attempt_folder(tmp_path, base, 100, 1) + assert first != second + assert (first / "k6.txt").read_text() == "failed attempt" + + +@pytest.mark.parametrize("recover", [False, True]) +def test_disturbed_runs_never_enter_summary( + scaling: ModuleType, monkeypatch: pytest.MonkeyPatch, recover: bool +) -> None: + args = SimpleNamespace( + base_env={}, out="unused", retries=1, pause=0, rates="100", rejected=Mock() + ) + rejected = {"raw_dir": "attempt1", "disturbed": True, "saturated": False} + accepted = {"raw_dir": "attempt2", "disturbed": False, "saturated": False} + monkeypatch.setattr(scaling, "layout", lambda *_: ({}, {})) + monkeypatch.setattr(scaling, "apply", lambda *_: {}) + monkeypatch.setattr(scaling, "deployment", lambda *_: []) + monkeypatch.setattr(scaling, "wait_quiet", lambda *_: None) + monkeypatch.setattr( + scaling, "step", Mock(side_effect=[rejected, accepted if recover else rejected]) + ) + output = Mock() + if recover: + scaling.block(args, output, "1x1", "base", [1]) + output.writerows.assert_called_once_with([accepted]) + args.rejected.writerow.assert_called_once_with(rejected) + else: + with pytest.raises(SystemExit, match="exhausted retries"): + scaling.block(args, output, "1x1", "base", [1]) + output.writerows.assert_not_called() + assert args.rejected.writerow.call_count == 2 diff --git a/tools/plot_track1.py b/tools/plot_track1.py index 957245d4..b208a19d 100644 --- a/tools/plot_track1.py +++ b/tools/plot_track1.py @@ -10,6 +10,7 @@ ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4") PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948") MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*") USL_POINTS = 3 +MIN_REPEATS = 3 COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6")) INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb" SUMMARY = ( @@ -57,9 +58,15 @@ def load(paths): groups = collections.defaultdict(lambda: collections.defaultdict(list)) for path in paths: with Path(path).open() as handle: - for row in csv.DictReader(handle): + for row in csv.DictReader(line for line in handle if not line.startswith("#")): + if row["disturbed"] == "True": + raise ValueError(f"disturbed run in accepted sample: {path}") key = (Path(path).parent.name, row["variant"], row["config"]) groups[key][int(row["rate"])].append(row) + for key, rates in groups.items(): + for rate, rows in rates.items(): + if len(rows) < MIN_REPEATS or len({row["repeat"] for row in rows}) != len(rows): + raise ValueError(f"need at least three distinct repeats: {key}, {rate}") return groups