forked from mtusi2026/mtusi2026lab2
118 lines
4.3 KiB
Python
118 lines
4.3 KiB
Python
"""date: 2026-11-06T08:41:01.934777+00:00 commit: b063c8780eabf63cdb9da3802f82440750308d6b"""
|
|
|
|
import csv
|
|
import datetime
|
|
import json
|
|
import pathlib
|
|
import statistics
|
|
import subprocess
|
|
|
|
root = pathlib.Path(__file__).resolve().parents[2]
|
|
base = root / "evidence/final-validation/current-after"
|
|
rows = list(
|
|
csv.DictReader(line for line in (base / "summary.csv").open() if not line.startswith("#"))
|
|
)
|
|
assert len(rows) == 9
|
|
summaries = []
|
|
for rate in [2400, 3600, 5400]:
|
|
group = [r for r in rows if int(r["rate"]) == rate]
|
|
assert sorted(int(r["repeat"]) for r in group) == [1, 2, 3]
|
|
for row in group:
|
|
folder = base / row["raw_dir"]
|
|
data = json.loads((folder / "k6/summary.json").read_text())
|
|
meta = json.loads((folder / "metadata.json").read_text())
|
|
assert meta["commit"] == "b063c8780eabf63cdb9da3802f82440750308d6b"
|
|
assert meta["warmup_seconds"] == 10 and meta["duration_seconds"] == 30
|
|
assert "setup_data" not in data
|
|
assert row["k6_exit"] == "0" and row["disturbed"] == "False"
|
|
assert float(row["cpu_other"]) <= 100 and float(row["gap_s"]) <= 10
|
|
assert float(row["wall_s"]) <= 130 and 0 < float(row["cpu_k6"]) < 285
|
|
apis = [c for c in meta["containers"] if "-api-" in c["name"]]
|
|
assert len(apis) == 8
|
|
for c in apis:
|
|
assert (c["NanoCpus"], c["Memory"], c["CpusetCpus"], c["PortBindings"]) == (
|
|
1e9,
|
|
512 * 2**20,
|
|
"7-14",
|
|
{},
|
|
)
|
|
for service, cpus, memory, pin in [
|
|
("postgres", 2, 1024, "0-1"),
|
|
("lb", 1, 128, "2"),
|
|
("valkey", 0.5, 128, "3"),
|
|
]:
|
|
c = next(c for c in meta["containers"] if "-" + service + "-" in c["name"])
|
|
assert (c["NanoCpus"], c["Memory"], c["CpusetCpus"]) == (
|
|
cpus * 1e9,
|
|
memory * 2**20,
|
|
pin,
|
|
)
|
|
metrics = data["metrics"]
|
|
|
|
def metric(name, key):
|
|
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(key, 0)
|
|
|
|
requests = metric("http_reqs", "count")
|
|
errors = metric("http_req_failed", "rate")
|
|
drops = metric("dropped_iterations", "count")
|
|
p95 = metric("http_req_duration", "p(95)")
|
|
expected = {
|
|
"requests": requests,
|
|
"ok_rps": requests * (1 - errors) / 30,
|
|
"rps": requests / 30,
|
|
"failed_ratio": errors,
|
|
"dropped_iterations": drops,
|
|
"p95_ms": p95,
|
|
"p99_ms": metric("http_req_duration", "p(99)"),
|
|
}
|
|
for key, value in expected.items():
|
|
assert abs(float(row[key]) - value) < (0.000006 if key == "failed_ratio" else 0.011)
|
|
saturated = round(errors, 5) > 0.01 or drops > 0.01 * rate * 30 or round(p95, 2) > 500
|
|
assert (row["saturated"] == "True") == saturated
|
|
for file in [
|
|
"docker_stats.csv",
|
|
"vm_stat.csv",
|
|
"pg_activity.csv",
|
|
"lb_stats.csv",
|
|
"pg_statements.csv",
|
|
]:
|
|
assert list(
|
|
csv.DictReader(line for line in (folder / file).open() if not line.startswith("#"))
|
|
)
|
|
item = {
|
|
"offered_rps": rate,
|
|
"repeats": 3,
|
|
"saturated_repeats": sum(r["saturated"] == "True" for r in group),
|
|
}
|
|
for field in [
|
|
"ok_rps",
|
|
"p95_ms",
|
|
"p99_ms",
|
|
"failed_ratio",
|
|
"dropped_iterations",
|
|
"cpu_api",
|
|
"cpu_postgres",
|
|
"cpu_lb",
|
|
"pg_lock_avg",
|
|
"lb_qcur_max",
|
|
]:
|
|
values = [float(r[field]) for r in group]
|
|
item[field] = {"median": statistics.median(values), "min": min(values), "max": max(values)}
|
|
summaries.append(item)
|
|
report = {
|
|
"date": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip(),
|
|
"runs": 9,
|
|
"requests": sum(int(r["requests"]) for r in rows),
|
|
"database_mb_range": [
|
|
min(float(r["db_mb"]) for r in rows),
|
|
max(float(r["db_mb"]) for r in rows),
|
|
],
|
|
"groups": summaries,
|
|
"quality_checks_pass": True,
|
|
}
|
|
(root / "evidence/final-validation/current-audit.json").write_text(
|
|
json.dumps(report, indent=2) + "\n"
|
|
)
|
|
print(json.dumps(report, indent=2))
|