Preserve scaling data and reject contaminated benchmark attempts
This commit is contained in:
@@ -11,9 +11,10 @@ RATE ?= 100
|
||||
FROM_REV ?=
|
||||
SHIFT ?= 0
|
||||
SCRIPTS_PY = alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools
|
||||
TRACK1 = evidence/track1
|
||||
SCALING = $(PYTHON) scripts/scaling.py --fresh-db
|
||||
BEFORE_REV ?= edc091d
|
||||
TRACK1 ?= evidence/track1-sqlalchemy
|
||||
SCALING = $(PYTHON) scripts/scaling.py
|
||||
BEFORE_REV ?= 5c8f605cd3bd40245109348463d3e7111379b2d3
|
||||
BASELINE = --env API_IMAGE=mtusi-messenger:$(BEFORE_REV)
|
||||
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
PYTEST_ARGS ?=
|
||||
|
||||
@@ -22,7 +23,9 @@ PYTEST_ARGS ?=
|
||||
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
|
||||
persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \
|
||||
limits detection failover-each rolling-update level4 \
|
||||
track1 track1-host track1-a track1-b track1-compare track1-control plots
|
||||
track1 track1-images track1-host track1-a track1-b track1-compare track1-control plots
|
||||
|
||||
.NOTPARALLEL:
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -128,30 +131,32 @@ endif
|
||||
|
||||
level4: limits lb-check detection failover-each rolling-update
|
||||
|
||||
track1: track1-host track1-a track1-b track1-compare track1-control plots
|
||||
track1: track1-images track1-host track1-a track1-b track1-compare track1-control plots
|
||||
|
||||
track1-images:
|
||||
git archive "$(BEFORE_REV)" | docker build -q -t mtusi-messenger:$(BEFORE_REV) -
|
||||
$(COMPOSE) build -q api
|
||||
|
||||
track1-host:
|
||||
sh scripts/host-snapshot.sh $(TRACK1)/host.txt
|
||||
|
||||
track1-a:
|
||||
$(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 \
|
||||
--start-per-cpu 150
|
||||
--start-per-cpu 150 $(BASELINE)
|
||||
|
||||
track1-b:
|
||||
$(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 \
|
||||
--start-per-cpu 450 --search 1.5:9000
|
||||
--start-per-cpu 300 --search 1.5:9000 $(BASELINE)
|
||||
|
||||
track1-compare:
|
||||
git archive "$(BEFORE_REV)" | docker build -q -t mtusi-messenger:$(BEFORE_REV) -
|
||||
$(COMPOSE) build -q api
|
||||
$(SCALING) --out $(TRACK1)/compare --configs 8x1 --api-memory 512 --rates 3600,5400,8100 \
|
||||
track1-compare: track1-images
|
||||
$(SCALING) --out $(TRACK1)/compare --configs 8x1 --api-memory 512 --rates 2400,3600,5400 \
|
||||
--variant before:API_IMAGE=mtusi-messenger:$(BEFORE_REV) --variant after
|
||||
|
||||
track1-control:
|
||||
$(SCALING) --out $(TRACK1)/control-pool --configs 8x1 --api-memory 512 --rates 5400 \
|
||||
$(SCALING) $(BASELINE) --out $(TRACK1)/control-pool --configs 8x1 --api-memory 512 --rates 5400 \
|
||||
--variant pool4:DB_POOL_MAX_SIZE=4 --variant pool8 --variant pool16:DB_POOL_MAX_SIZE=16 \
|
||||
--variant maxconn16:API_SERVER_MAXCONN=16 --variant maxconn64:API_SERVER_MAXCONN=64
|
||||
$(SCALING) --out $(TRACK1)/control-cpus --configs 4x1,8x0.5 --api-memory 512 --rates 4560,6830 \
|
||||
$(SCALING) $(BASELINE) --out $(TRACK1)/control-cpus --configs 4x1,8x0.5 --api-memory-total 2048 --rates 3040,4560 \
|
||||
--variant pinned
|
||||
|
||||
plots:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
## 1. Методика и безопасная обвязка
|
||||
|
||||
- [ ] 1.1 Проверить пилот, исключить удаление volumes и перезапись попыток, сохранять provenance и реальные лимиты.
|
||||
- [ ] 1.2 Проверить обвязку собственными тестами, make check/lint/student-test и smoke.
|
||||
- [x] 1.1 Проверить пилот, исключить удаление volumes и перезапись попыток, сохранять provenance и реальные лимиты.
|
||||
- [x] 1.2 Проверить обвязку собственными тестами, make check/lint/student-test и smoke.
|
||||
|
||||
## 2. Базовые серии SQLAlchemy
|
||||
|
||||
|
||||
+75
-7
@@ -28,7 +28,7 @@ PINNED = {
|
||||
}
|
||||
API_FIRST_CPU = 7
|
||||
FIELDS = (
|
||||
"variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
|
||||
"raw_dir variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
|
||||
"started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio "
|
||||
"dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres "
|
||||
"cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max "
|
||||
@@ -228,6 +228,58 @@ def apply(base, settings, env, out):
|
||||
return env
|
||||
|
||||
|
||||
def deployment(env):
|
||||
ids = compose("ps", "-q", env=env).split()
|
||||
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", template, *ids],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
containers = []
|
||||
for line in result.stdout.splitlines():
|
||||
name, image, config = line.split(" ", 2)
|
||||
limits = json.loads(config)
|
||||
containers.append(
|
||||
dict(
|
||||
name=json.loads(name),
|
||||
image=json.loads(image),
|
||||
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
|
||||
)
|
||||
)
|
||||
return containers
|
||||
|
||||
|
||||
def provenance(folder, args, env):
|
||||
stamp = time.strftime("%FT%TZ", time.gmtime())
|
||||
metadata = {
|
||||
"date": stamp,
|
||||
"commit": args.commit,
|
||||
"project": args.project,
|
||||
"command": sys.argv,
|
||||
"api_image": env.get("API_IMAGE", "compose default"),
|
||||
"containers": args.deployment,
|
||||
}
|
||||
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
|
||||
for path in folder.rglob("*"):
|
||||
if path.suffix in {".csv", ".txt"}:
|
||||
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
|
||||
elif path.name == "summary.json":
|
||||
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
|
||||
path.write_text(json.dumps(data) + "\n")
|
||||
|
||||
|
||||
def attempt_folder(out, base, rate, repeat):
|
||||
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
||||
attempt = 1
|
||||
while (Path(out) / f"{name}_attempt{attempt}").exists():
|
||||
attempt += 1
|
||||
folder = Path(out) / f"{name}_attempt{attempt}"
|
||||
folder.mkdir(parents=True)
|
||||
return folder
|
||||
|
||||
|
||||
def measure(folder, args, rate, env):
|
||||
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
|
||||
database = subprocess.run(
|
||||
@@ -255,8 +307,7 @@ def measure(folder, args, rate, env):
|
||||
|
||||
|
||||
def step(base, rate, repeat, args, env):
|
||||
folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
folder = attempt_folder(args.out, base, rate, repeat)
|
||||
code, start, database, wall = measure(folder, args, rate, env)
|
||||
end = start + args.duration
|
||||
summary = folder / "k6/summary.json"
|
||||
@@ -266,7 +317,15 @@ def step(base, rate, repeat, args, env):
|
||||
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
|
||||
or metrics["p95_ms"] > args.slo_p95_ms
|
||||
)
|
||||
row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics)
|
||||
row = dict(
|
||||
base,
|
||||
raw_dir=folder.name,
|
||||
rate=rate,
|
||||
repeat=repeat,
|
||||
k6_exit=code,
|
||||
saturated=saturated,
|
||||
**metrics,
|
||||
)
|
||||
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
|
||||
row["db_mb"] = round(int(database[0]) / 2**20, 1)
|
||||
row["messages"] = int(database[1])
|
||||
@@ -284,7 +343,13 @@ def step(base, rate, repeat, args, env):
|
||||
row["cpu_other"] > args.max_other
|
||||
or row["gap_s"] > args.max_gap
|
||||
or wall > args.warmup + args.duration + args.max_overhead
|
||||
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
|
||||
or not rows(folder / "pg_activity.csv", start, end)
|
||||
or not rows(folder / "lb_stats.csv", start, end)
|
||||
)
|
||||
provenance(folder, args, env)
|
||||
if code or not metrics["requests"]:
|
||||
raise SystemExit(f"invalid k6 run: {folder}")
|
||||
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
||||
return row
|
||||
|
||||
@@ -310,7 +375,6 @@ def arguments():
|
||||
action="append",
|
||||
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
|
||||
)
|
||||
parser.add_argument("--fresh-db", action="store_true", help="drop volumes before a config")
|
||||
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
|
||||
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
|
||||
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
|
||||
@@ -344,6 +408,7 @@ def arguments():
|
||||
def block(args, output, config, variant, repeats):
|
||||
base, settings = layout(config, args, variant)
|
||||
current, streak = apply(base, settings, args.base_env, args.out), 0
|
||||
args.deployment = deployment(current)
|
||||
for rate in rates(args, base):
|
||||
results = []
|
||||
for repeat in repeats:
|
||||
@@ -354,6 +419,8 @@ def block(args, output, config, variant, repeats):
|
||||
break
|
||||
args.rejected.writerow(row)
|
||||
wait_quiet(args)
|
||||
else:
|
||||
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
|
||||
results.append(row)
|
||||
output.writerows(results)
|
||||
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
||||
@@ -364,8 +431,6 @@ def block(args, output, config, variant, repeats):
|
||||
def series(args, output):
|
||||
every = range(args.first_repeat, args.first_repeat + args.repeats)
|
||||
for config in args.configs.split(","):
|
||||
if args.fresh_db:
|
||||
compose("down", "-v", env=args.base_env)
|
||||
if len(args.variants) == 1:
|
||||
block(args, output, config, args.variants[0], every)
|
||||
continue
|
||||
@@ -380,6 +445,8 @@ def table(path):
|
||||
with path.open("a", newline="", buffering=1) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
||||
if fresh:
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
|
||||
writer.writeheader()
|
||||
yield writer
|
||||
|
||||
@@ -389,6 +456,7 @@ def main():
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
||||
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
||||
try:
|
||||
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scaling(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
|
||||
scripts = Path(__file__).resolve().parents[2] / "scripts"
|
||||
monkeypatch.syspath_prepend(str(scripts))
|
||||
spec = importlib.util.spec_from_file_location("scaling", scripts / "scaling.py")
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_retry_preserves_previous_raw_data(scaling: ModuleType, tmp_path: Path) -> None:
|
||||
base = {"variant": "base", "config": "1x1"}
|
||||
first = scaling.attempt_folder(tmp_path, base, 100, 1)
|
||||
(first / "k6.txt").write_text("failed attempt")
|
||||
second = scaling.attempt_folder(tmp_path, base, 100, 1)
|
||||
assert first != second
|
||||
assert (first / "k6.txt").read_text() == "failed attempt"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("recover", [False, True])
|
||||
def test_disturbed_runs_never_enter_summary(
|
||||
scaling: ModuleType, monkeypatch: pytest.MonkeyPatch, recover: bool
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
base_env={}, out="unused", retries=1, pause=0, rates="100", rejected=Mock()
|
||||
)
|
||||
rejected = {"raw_dir": "attempt1", "disturbed": True, "saturated": False}
|
||||
accepted = {"raw_dir": "attempt2", "disturbed": False, "saturated": False}
|
||||
monkeypatch.setattr(scaling, "layout", lambda *_: ({}, {}))
|
||||
monkeypatch.setattr(scaling, "apply", lambda *_: {})
|
||||
monkeypatch.setattr(scaling, "deployment", lambda *_: [])
|
||||
monkeypatch.setattr(scaling, "wait_quiet", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
scaling, "step", Mock(side_effect=[rejected, accepted if recover else rejected])
|
||||
)
|
||||
output = Mock()
|
||||
if recover:
|
||||
scaling.block(args, output, "1x1", "base", [1])
|
||||
output.writerows.assert_called_once_with([accepted])
|
||||
args.rejected.writerow.assert_called_once_with(rejected)
|
||||
else:
|
||||
with pytest.raises(SystemExit, match="exhausted retries"):
|
||||
scaling.block(args, output, "1x1", "base", [1])
|
||||
output.writerows.assert_not_called()
|
||||
assert args.rejected.writerow.call_count == 2
|
||||
@@ -10,6 +10,7 @@ ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4")
|
||||
PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948")
|
||||
MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*")
|
||||
USL_POINTS = 3
|
||||
MIN_REPEATS = 3
|
||||
COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6"))
|
||||
INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb"
|
||||
SUMMARY = (
|
||||
@@ -57,9 +58,15 @@ def load(paths):
|
||||
groups = collections.defaultdict(lambda: collections.defaultdict(list))
|
||||
for path in paths:
|
||||
with Path(path).open() as handle:
|
||||
for row in csv.DictReader(handle):
|
||||
for row in csv.DictReader(line for line in handle if not line.startswith("#")):
|
||||
if row["disturbed"] == "True":
|
||||
raise ValueError(f"disturbed run in accepted sample: {path}")
|
||||
key = (Path(path).parent.name, row["variant"], row["config"])
|
||||
groups[key][int(row["rate"])].append(row)
|
||||
for key, rates in groups.items():
|
||||
for rate, rows in rates.items():
|
||||
if len(rows) < MIN_REPEATS or len({row["repeat"] for row in rows}) != len(rows):
|
||||
raise ValueError(f"need at least three distinct repeats: {key}, {rate}")
|
||||
return groups
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user