Preserve scaling data and reject contaminated benchmark attempts

This commit is contained in:
Artyom
2026-11-05 22:42:51 +03:00
parent 74a39fa93e
commit 05d69ba6ac
5 changed files with 157 additions and 23 deletions
+18 -13
View File
@@ -11,9 +11,10 @@ RATE ?= 100
FROM_REV ?=
SHIFT ?= 0
SCRIPTS_PY = alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools
TRACK1 = evidence/track1
SCALING = $(PYTHON) scripts/scaling.py --fresh-db
BEFORE_REV ?= edc091d
TRACK1 ?= evidence/track1-sqlalchemy
SCALING = $(PYTHON) scripts/scaling.py
BEFORE_REV ?= 5c8f605cd3bd40245109348463d3e7111379b2d3
BASELINE = --env API_IMAGE=mtusi-messenger:$(BEFORE_REV)
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
PYTEST_ARGS ?=
@@ -22,7 +23,9 @@ PYTEST_ARGS ?=
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \
limits detection failover-each rolling-update level4 \
track1 track1-host track1-a track1-b track1-compare track1-control plots
track1 track1-images track1-host track1-a track1-b track1-compare track1-control plots
.NOTPARALLEL:
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -128,30 +131,32 @@ endif
level4: limits lb-check detection failover-each rolling-update
track1: track1-host track1-a track1-b track1-compare track1-control plots
track1: track1-images track1-host track1-a track1-b track1-compare track1-control plots
track1-images:
git archive "$(BEFORE_REV)" | docker build -q -t mtusi-messenger:$(BEFORE_REV) -
$(COMPOSE) build -q api
track1-host:
sh scripts/host-snapshot.sh $(TRACK1)/host.txt
track1-a:
$(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 \
--start-per-cpu 150
--start-per-cpu 150 $(BASELINE)
track1-b:
$(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 \
--start-per-cpu 450 --search 1.5:9000
--start-per-cpu 300 --search 1.5:9000 $(BASELINE)
track1-compare:
git archive "$(BEFORE_REV)" | docker build -q -t mtusi-messenger:$(BEFORE_REV) -
$(COMPOSE) build -q api
$(SCALING) --out $(TRACK1)/compare --configs 8x1 --api-memory 512 --rates 3600,5400,8100 \
track1-compare: track1-images
$(SCALING) --out $(TRACK1)/compare --configs 8x1 --api-memory 512 --rates 2400,3600,5400 \
--variant before:API_IMAGE=mtusi-messenger:$(BEFORE_REV) --variant after
track1-control:
$(SCALING) --out $(TRACK1)/control-pool --configs 8x1 --api-memory 512 --rates 5400 \
$(SCALING) $(BASELINE) --out $(TRACK1)/control-pool --configs 8x1 --api-memory 512 --rates 5400 \
--variant pool4:DB_POOL_MAX_SIZE=4 --variant pool8 --variant pool16:DB_POOL_MAX_SIZE=16 \
--variant maxconn16:API_SERVER_MAXCONN=16 --variant maxconn64:API_SERVER_MAXCONN=64
$(SCALING) --out $(TRACK1)/control-cpus --configs 4x1,8x0.5 --api-memory 512 --rates 4560,6830 \
$(SCALING) $(BASELINE) --out $(TRACK1)/control-cpus --configs 4x1,8x0.5 --api-memory-total 2048 --rates 3040,4560 \
--variant pinned
plots:
@@ -1,7 +1,7 @@
## 1. Методика и безопасная обвязка
- [ ] 1.1 Проверить пилот, исключить удаление volumes и перезапись попыток, сохранять provenance и реальные лимиты.
- [ ] 1.2 Проверить обвязку собственными тестами, make check/lint/student-test и smoke.
- [x] 1.1 Проверить пилот, исключить удаление volumes и перезапись попыток, сохранять provenance и реальные лимиты.
- [x] 1.2 Проверить обвязку собственными тестами, make check/lint/student-test и smoke.
## 2. Базовые серии SQLAlchemy
+75 -7
View File
@@ -28,7 +28,7 @@ PINNED = {
}
API_FIRST_CPU = 7
FIELDS = (
"variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
"raw_dir variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
"started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio "
"dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres "
"cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max "
@@ -228,6 +228,58 @@ def apply(base, settings, env, out):
return env
def deployment(env):
ids = compose("ps", "-q", env=env).split()
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
result = subprocess.run(
["docker", "inspect", "--format", template, *ids],
check=True,
capture_output=True,
text=True,
)
containers = []
for line in result.stdout.splitlines():
name, image, config = line.split(" ", 2)
limits = json.loads(config)
containers.append(
dict(
name=json.loads(name),
image=json.loads(image),
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
)
)
return containers
def provenance(folder, args, env):
stamp = time.strftime("%FT%TZ", time.gmtime())
metadata = {
"date": stamp,
"commit": args.commit,
"project": args.project,
"command": sys.argv,
"api_image": env.get("API_IMAGE", "compose default"),
"containers": args.deployment,
}
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
for path in folder.rglob("*"):
if path.suffix in {".csv", ".txt"}:
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
elif path.name == "summary.json":
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
path.write_text(json.dumps(data) + "\n")
def attempt_folder(out, base, rate, repeat):
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
attempt = 1
while (Path(out) / f"{name}_attempt{attempt}").exists():
attempt += 1
folder = Path(out) / f"{name}_attempt{attempt}"
folder.mkdir(parents=True)
return folder
def measure(folder, args, rate, env):
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
database = subprocess.run(
@@ -255,8 +307,7 @@ def measure(folder, args, rate, env):
def step(base, rate, repeat, args, env):
folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
folder.mkdir(parents=True, exist_ok=True)
folder = attempt_folder(args.out, base, rate, repeat)
code, start, database, wall = measure(folder, args, rate, env)
end = start + args.duration
summary = folder / "k6/summary.json"
@@ -266,7 +317,15 @@ def step(base, rate, repeat, args, env):
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
or metrics["p95_ms"] > args.slo_p95_ms
)
row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics)
row = dict(
base,
raw_dir=folder.name,
rate=rate,
repeat=repeat,
k6_exit=code,
saturated=saturated,
**metrics,
)
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
row["db_mb"] = round(int(database[0]) / 2**20, 1)
row["messages"] = int(database[1])
@@ -284,7 +343,13 @@ def step(base, rate, repeat, args, env):
row["cpu_other"] > args.max_other
or row["gap_s"] > args.max_gap
or wall > args.warmup + args.duration + args.max_overhead
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
or not rows(folder / "pg_activity.csv", start, end)
or not rows(folder / "lb_stats.csv", start, end)
)
provenance(folder, args, env)
if code or not metrics["requests"]:
raise SystemExit(f"invalid k6 run: {folder}")
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
return row
@@ -310,7 +375,6 @@ def arguments():
action="append",
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
)
parser.add_argument("--fresh-db", action="store_true", help="drop volumes before a config")
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
@@ -344,6 +408,7 @@ def arguments():
def block(args, output, config, variant, repeats):
base, settings = layout(config, args, variant)
current, streak = apply(base, settings, args.base_env, args.out), 0
args.deployment = deployment(current)
for rate in rates(args, base):
results = []
for repeat in repeats:
@@ -354,6 +419,8 @@ def block(args, output, config, variant, repeats):
break
args.rejected.writerow(row)
wait_quiet(args)
else:
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
results.append(row)
output.writerows(results)
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
@@ -364,8 +431,6 @@ def block(args, output, config, variant, repeats):
def series(args, output):
every = range(args.first_repeat, args.first_repeat + args.repeats)
for config in args.configs.split(","):
if args.fresh_db:
compose("down", "-v", env=args.base_env)
if len(args.variants) == 1:
block(args, output, config, args.variants[0], every)
continue
@@ -380,6 +445,8 @@ def table(path):
with path.open("a", newline="", buffering=1) as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
if fresh:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
writer.writeheader()
yield writer
@@ -389,6 +456,7 @@ def main():
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
args.project = json.loads(compose("config", "--format", "json"))["name"]
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
try:
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
+54
View File
@@ -0,0 +1,54 @@
import importlib.util
from pathlib import Path
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
@pytest.fixture
def scaling(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
scripts = Path(__file__).resolve().parents[2] / "scripts"
monkeypatch.syspath_prepend(str(scripts))
spec = importlib.util.spec_from_file_location("scaling", scripts / "scaling.py")
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_retry_preserves_previous_raw_data(scaling: ModuleType, tmp_path: Path) -> None:
base = {"variant": "base", "config": "1x1"}
first = scaling.attempt_folder(tmp_path, base, 100, 1)
(first / "k6.txt").write_text("failed attempt")
second = scaling.attempt_folder(tmp_path, base, 100, 1)
assert first != second
assert (first / "k6.txt").read_text() == "failed attempt"
@pytest.mark.parametrize("recover", [False, True])
def test_disturbed_runs_never_enter_summary(
scaling: ModuleType, monkeypatch: pytest.MonkeyPatch, recover: bool
) -> None:
args = SimpleNamespace(
base_env={}, out="unused", retries=1, pause=0, rates="100", rejected=Mock()
)
rejected = {"raw_dir": "attempt1", "disturbed": True, "saturated": False}
accepted = {"raw_dir": "attempt2", "disturbed": False, "saturated": False}
monkeypatch.setattr(scaling, "layout", lambda *_: ({}, {}))
monkeypatch.setattr(scaling, "apply", lambda *_: {})
monkeypatch.setattr(scaling, "deployment", lambda *_: [])
monkeypatch.setattr(scaling, "wait_quiet", lambda *_: None)
monkeypatch.setattr(
scaling, "step", Mock(side_effect=[rejected, accepted if recover else rejected])
)
output = Mock()
if recover:
scaling.block(args, output, "1x1", "base", [1])
output.writerows.assert_called_once_with([accepted])
args.rejected.writerow.assert_called_once_with(rejected)
else:
with pytest.raises(SystemExit, match="exhausted retries"):
scaling.block(args, output, "1x1", "base", [1])
output.writerows.assert_not_called()
assert args.rejected.writerow.call_count == 2
+8 -1
View File
@@ -10,6 +10,7 @@ ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4")
PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948")
MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*")
USL_POINTS = 3
MIN_REPEATS = 3
COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6"))
INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb"
SUMMARY = (
@@ -57,9 +58,15 @@ def load(paths):
groups = collections.defaultdict(lambda: collections.defaultdict(list))
for path in paths:
with Path(path).open() as handle:
for row in csv.DictReader(handle):
for row in csv.DictReader(line for line in handle if not line.startswith("#")):
if row["disturbed"] == "True":
raise ValueError(f"disturbed run in accepted sample: {path}")
key = (Path(path).parent.name, row["variant"], row["config"])
groups[key][int(row["rate"])].append(row)
for key, rates in groups.items():
for rate, rows in rates.items():
if len(rows) < MIN_REPEATS or len({row["repeat"] for row in rows}) != len(rows):
raise ValueError(f"need at least three distinct repeats: {key}, {rate}")
return groups