Files

167 lines
6.7 KiB
Python

"""date: 2026-11-06T08:33:49.179027+00:00 commit: b063c8780eabf63cdb9da3802f82440750308d6b"""
import collections
import csv
import datetime
import hashlib
import json
import pathlib
import statistics
import struct
import subprocess
import zlib
root = pathlib.Path(__file__).resolve().parents[2]
base = root / "evidence/track1-sqlalchemy"
sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
def table(path):
return list(csv.DictReader(line for line in path.open() if not line.startswith("#")))
series = {}
groups = collections.defaultdict(list)
images = collections.defaultdict(set)
for path in sorted(base.glob("*/summary.csv")):
rows = table(path)
series[path.parent.name] = len(rows)
for row in rows:
folder = path.parent / row["raw_dir"]
meta = json.loads((folder / "metadata.json").read_text())
data = json.loads((folder / "k6/summary.json").read_text())
assert "setup_data" not in data
for option, default in [("warmup", 10), ("duration", 30)]:
if option + "_seconds" in meta:
actual = meta[option + "_seconds"]
elif "--" + option in meta["command"]:
actual = int(meta["command"][meta["command"].index("--" + option) + 1])
else:
historical = subprocess.check_output(
["git", "show", meta["commit"] + ":scripts/scaling.py"], cwd=root, text=True
)
assert (
f'parser.add_argument("--{option}", type=int, default={default})' in historical
)
actual = default
assert actual == default
assert row["disturbed"] == "False" and row["k6_exit"] == "0"
metrics = data["metrics"]
def metric(name, key):
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(key, 0)
count = metric("http_reqs", "count")
errors = metric("http_req_failed", "rate")
dropped = metric("dropped_iterations", "count")
p95 = metric("http_req_duration", "p(95)")
expected = {
"requests": count,
"ok_rps": count * (1 - errors) / 30,
"rps": count / 30,
"failed_ratio": errors,
"dropped_iterations": dropped,
"p50_ms": metric("http_req_duration", "med"),
"p95_ms": p95,
"p99_ms": metric("http_req_duration", "p(99)"),
}
for key, value in expected.items():
assert abs(float(row[key]) - value) < (0.000006 if key == "failed_ratio" else 0.011), (
folder,
key,
)
saturated = (
round(errors, 5) > 0.01 or dropped > 0.01 * int(row["rate"]) * 30 or round(p95, 2) > 500
)
assert (row["saturated"] == "True") == saturated
assert float(row["cpu_other"]) <= 100 and float(row["gap_s"]) <= 10
assert float(row["wall_s"]) <= 130
for resource in ["api", "postgres", "k6", "vm"]:
assert float(row["cpu_" + resource]) > 0
for file in [
"docker_stats.csv",
"vm_stat.csv",
"lb_stats.csv",
"pg_activity.csv",
"pg_statements.csv",
]:
assert table(folder / file), (folder, file)
apis = [c for c in meta["containers"] if "-api-" in c["name"]]
assert len(apis) == int(row["replicas"])
for c in apis:
assert c["NanoCpus"] == float(row["cpus"]) * 1e9
assert c["Memory"] == int(row["memory_mb"]) * 2**20
assert c["CpusetCpus"] == row["api_cpuset"]
assert not c["PortBindings"]
images[(path.parent.name, row["variant"])].add(c["image"])
for service, cpus, memory, pin in [
("postgres", 2, 1024, "0-1"),
("lb", 1, 128, "2"),
("valkey", 0.5, 128, "3"),
]:
c = next(c for c in meta["containers"] if "-" + service + "-" in c["name"])
assert (c["NanoCpus"], c["Memory"], c["CpusetCpus"]) == (
cpus * 1e9,
memory * 2**20,
pin,
)
groups[(path.parent.name, row["variant"], row["config"], int(row["rate"]))].append(row)
for key, rows in groups.items():
assert sorted(int(r["repeat"]) for r in rows) == [1, 2, 3], key
capacities = table(base / "plots/capacity.csv")
for capacity in capacities:
key = (capacity["series"], capacity["variant"], capacity["config"])
rates = {k[3]: v for k, v in groups.items() if k[:3] == key}
peak = max(rates, key=lambda rate: statistics.median(float(r["ok_rps"]) for r in rates[rate]))
assert peak == int(capacity["peak_rate"])
assert statistics.median(float(r["ok_rps"]) for r in rates[peak]) == float(
capacity["peak_ok_rps"]
)
good = [rate for rate, rows in rates.items() if sum(r["saturated"] == "True" for r in rows) < 2]
assert (max(good) if good else "") == (
int(capacity["capacity_rate"]) if capacity["capacity_rate"] else ""
)
pngs = []
for p in sorted((base / "plots").glob("*.png")):
raw = p.read_bytes()
assert raw[:8] == b"\x89PNG\r\n\x1a\n"
offset = 8
while offset < len(raw):
size = struct.unpack(">I", raw[offset : offset + 4])[0]
payload = raw[offset + 4 : offset + 8 + size]
crc = struct.unpack(">I", raw[offset + 8 + size : offset + 12 + size])[0]
assert zlib.crc32(payload) == crc
offset += size + 12
pngs.append({"file": str(p.relative_to(root)), "sha256": hashlib.sha256(raw).hexdigest()})
assert sum(series.values()) == 135 and len(capacities) == 15 and len(pngs) == 12
all_rows = [row for rows in groups.values() for row in rows]
report = {
"date": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"commit": sha,
"accepted_series_counts": series,
"total_measurements": len(all_rows),
"triplicate_groups": len(groups),
"capacity_rows_recomputed": len(capacities),
"raw_k6_metrics_match": True,
"actual_resource_limits_match": True,
"disturbance_rules_pass": True,
"database_mb_range": [
min(float(r["db_mb"]) for r in all_rows),
max(float(r["db_mb"]) for r in all_rows),
],
"measured_requests": sum(int(r["requests"]) for r in all_rows),
"images_by_series_variant": {"/".join(k): sorted(v) for k, v in images.items()},
"png_integrity": pngs,
"pass": True,
}
(root / "evidence/final-validation/existing-evidence-audit.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ["png_integrity", "images_by_series_variant"]},
ensure_ascii=False,
indent=2,
)
)