Files

118 lines
4.3 KiB
Python

"""date: 2026-11-06T08:41:01.934777+00:00 commit: b063c8780eabf63cdb9da3802f82440750308d6b"""
import csv
import datetime
import json
import pathlib
import statistics
import subprocess
root = pathlib.Path(__file__).resolve().parents[2]
base = root / "evidence/final-validation/current-after"
rows = list(
csv.DictReader(line for line in (base / "summary.csv").open() if not line.startswith("#"))
)
assert len(rows) == 9
summaries = []
for rate in [2400, 3600, 5400]:
group = [r for r in rows if int(r["rate"]) == rate]
assert sorted(int(r["repeat"]) for r in group) == [1, 2, 3]
for row in group:
folder = base / row["raw_dir"]
data = json.loads((folder / "k6/summary.json").read_text())
meta = json.loads((folder / "metadata.json").read_text())
assert meta["commit"] == "b063c8780eabf63cdb9da3802f82440750308d6b"
assert meta["warmup_seconds"] == 10 and meta["duration_seconds"] == 30
assert "setup_data" not in data
assert row["k6_exit"] == "0" and row["disturbed"] == "False"
assert float(row["cpu_other"]) <= 100 and float(row["gap_s"]) <= 10
assert float(row["wall_s"]) <= 130 and 0 < float(row["cpu_k6"]) < 285
apis = [c for c in meta["containers"] if "-api-" in c["name"]]
assert len(apis) == 8
for c in apis:
assert (c["NanoCpus"], c["Memory"], c["CpusetCpus"], c["PortBindings"]) == (
1e9,
512 * 2**20,
"7-14",
{},
)
for service, cpus, memory, pin in [
("postgres", 2, 1024, "0-1"),
("lb", 1, 128, "2"),
("valkey", 0.5, 128, "3"),
]:
c = next(c for c in meta["containers"] if "-" + service + "-" in c["name"])
assert (c["NanoCpus"], c["Memory"], c["CpusetCpus"]) == (
cpus * 1e9,
memory * 2**20,
pin,
)
metrics = data["metrics"]
def metric(name, key):
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(key, 0)
requests = metric("http_reqs", "count")
errors = metric("http_req_failed", "rate")
drops = metric("dropped_iterations", "count")
p95 = metric("http_req_duration", "p(95)")
expected = {
"requests": requests,
"ok_rps": requests * (1 - errors) / 30,
"rps": requests / 30,
"failed_ratio": errors,
"dropped_iterations": drops,
"p95_ms": p95,
"p99_ms": metric("http_req_duration", "p(99)"),
}
for key, value in expected.items():
assert abs(float(row[key]) - value) < (0.000006 if key == "failed_ratio" else 0.011)
saturated = round(errors, 5) > 0.01 or drops > 0.01 * rate * 30 or round(p95, 2) > 500
assert (row["saturated"] == "True") == saturated
for file in [
"docker_stats.csv",
"vm_stat.csv",
"pg_activity.csv",
"lb_stats.csv",
"pg_statements.csv",
]:
assert list(
csv.DictReader(line for line in (folder / file).open() if not line.startswith("#"))
)
item = {
"offered_rps": rate,
"repeats": 3,
"saturated_repeats": sum(r["saturated"] == "True" for r in group),
}
for field in [
"ok_rps",
"p95_ms",
"p99_ms",
"failed_ratio",
"dropped_iterations",
"cpu_api",
"cpu_postgres",
"cpu_lb",
"pg_lock_avg",
"lb_qcur_max",
]:
values = [float(r[field]) for r in group]
item[field] = {"median": statistics.median(values), "min": min(values), "max": max(values)}
summaries.append(item)
report = {
"date": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip(),
"runs": 9,
"requests": sum(int(r["requests"]) for r in rows),
"database_mb_range": [
min(float(r["db_mb"]) for r in rows),
max(float(r["db_mb"]) for r in rows),
],
"groups": summaries,
"quality_checks_pass": True,
}
(root / "evidence/final-validation/current-audit.json").write_text(
json.dumps(report, indent=2) + "\n"
)
print(json.dumps(report, indent=2))