forked from mtusi2026/mtusi2026lab2
167 lines
6.7 KiB
Python
167 lines
6.7 KiB
Python
"""date: 2026-11-06T08:33:49.179027+00:00 commit: b063c8780eabf63cdb9da3802f82440750308d6b"""
|
|
|
|
import collections
|
|
import csv
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import zlib
|
|
|
|
root = pathlib.Path(__file__).resolve().parents[2]
|
|
base = root / "evidence/track1-sqlalchemy"
|
|
sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
|
|
|
|
|
|
def table(path):
|
|
return list(csv.DictReader(line for line in path.open() if not line.startswith("#")))
|
|
|
|
|
|
series = {}
|
|
groups = collections.defaultdict(list)
|
|
images = collections.defaultdict(set)
|
|
for path in sorted(base.glob("*/summary.csv")):
|
|
rows = table(path)
|
|
series[path.parent.name] = len(rows)
|
|
for row in rows:
|
|
folder = path.parent / row["raw_dir"]
|
|
meta = json.loads((folder / "metadata.json").read_text())
|
|
data = json.loads((folder / "k6/summary.json").read_text())
|
|
assert "setup_data" not in data
|
|
for option, default in [("warmup", 10), ("duration", 30)]:
|
|
if option + "_seconds" in meta:
|
|
actual = meta[option + "_seconds"]
|
|
elif "--" + option in meta["command"]:
|
|
actual = int(meta["command"][meta["command"].index("--" + option) + 1])
|
|
else:
|
|
historical = subprocess.check_output(
|
|
["git", "show", meta["commit"] + ":scripts/scaling.py"], cwd=root, text=True
|
|
)
|
|
assert (
|
|
f'parser.add_argument("--{option}", type=int, default={default})' in historical
|
|
)
|
|
actual = default
|
|
assert actual == default
|
|
assert row["disturbed"] == "False" and row["k6_exit"] == "0"
|
|
metrics = data["metrics"]
|
|
|
|
def metric(name, key):
|
|
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(key, 0)
|
|
|
|
count = metric("http_reqs", "count")
|
|
errors = metric("http_req_failed", "rate")
|
|
dropped = metric("dropped_iterations", "count")
|
|
p95 = metric("http_req_duration", "p(95)")
|
|
expected = {
|
|
"requests": count,
|
|
"ok_rps": count * (1 - errors) / 30,
|
|
"rps": count / 30,
|
|
"failed_ratio": errors,
|
|
"dropped_iterations": dropped,
|
|
"p50_ms": metric("http_req_duration", "med"),
|
|
"p95_ms": p95,
|
|
"p99_ms": metric("http_req_duration", "p(99)"),
|
|
}
|
|
for key, value in expected.items():
|
|
assert abs(float(row[key]) - value) < (0.000006 if key == "failed_ratio" else 0.011), (
|
|
folder,
|
|
key,
|
|
)
|
|
saturated = (
|
|
round(errors, 5) > 0.01 or dropped > 0.01 * int(row["rate"]) * 30 or round(p95, 2) > 500
|
|
)
|
|
assert (row["saturated"] == "True") == saturated
|
|
assert float(row["cpu_other"]) <= 100 and float(row["gap_s"]) <= 10
|
|
assert float(row["wall_s"]) <= 130
|
|
for resource in ["api", "postgres", "k6", "vm"]:
|
|
assert float(row["cpu_" + resource]) > 0
|
|
for file in [
|
|
"docker_stats.csv",
|
|
"vm_stat.csv",
|
|
"lb_stats.csv",
|
|
"pg_activity.csv",
|
|
"pg_statements.csv",
|
|
]:
|
|
assert table(folder / file), (folder, file)
|
|
apis = [c for c in meta["containers"] if "-api-" in c["name"]]
|
|
assert len(apis) == int(row["replicas"])
|
|
for c in apis:
|
|
assert c["NanoCpus"] == float(row["cpus"]) * 1e9
|
|
assert c["Memory"] == int(row["memory_mb"]) * 2**20
|
|
assert c["CpusetCpus"] == row["api_cpuset"]
|
|
assert not c["PortBindings"]
|
|
images[(path.parent.name, row["variant"])].add(c["image"])
|
|
for service, cpus, memory, pin in [
|
|
("postgres", 2, 1024, "0-1"),
|
|
("lb", 1, 128, "2"),
|
|
("valkey", 0.5, 128, "3"),
|
|
]:
|
|
c = next(c for c in meta["containers"] if "-" + service + "-" in c["name"])
|
|
assert (c["NanoCpus"], c["Memory"], c["CpusetCpus"]) == (
|
|
cpus * 1e9,
|
|
memory * 2**20,
|
|
pin,
|
|
)
|
|
groups[(path.parent.name, row["variant"], row["config"], int(row["rate"]))].append(row)
|
|
for key, rows in groups.items():
|
|
assert sorted(int(r["repeat"]) for r in rows) == [1, 2, 3], key
|
|
capacities = table(base / "plots/capacity.csv")
|
|
for capacity in capacities:
|
|
key = (capacity["series"], capacity["variant"], capacity["config"])
|
|
rates = {k[3]: v for k, v in groups.items() if k[:3] == key}
|
|
peak = max(rates, key=lambda rate: statistics.median(float(r["ok_rps"]) for r in rates[rate]))
|
|
assert peak == int(capacity["peak_rate"])
|
|
assert statistics.median(float(r["ok_rps"]) for r in rates[peak]) == float(
|
|
capacity["peak_ok_rps"]
|
|
)
|
|
good = [rate for rate, rows in rates.items() if sum(r["saturated"] == "True" for r in rows) < 2]
|
|
assert (max(good) if good else "") == (
|
|
int(capacity["capacity_rate"]) if capacity["capacity_rate"] else ""
|
|
)
|
|
pngs = []
|
|
for p in sorted((base / "plots").glob("*.png")):
|
|
raw = p.read_bytes()
|
|
assert raw[:8] == b"\x89PNG\r\n\x1a\n"
|
|
offset = 8
|
|
while offset < len(raw):
|
|
size = struct.unpack(">I", raw[offset : offset + 4])[0]
|
|
payload = raw[offset + 4 : offset + 8 + size]
|
|
crc = struct.unpack(">I", raw[offset + 8 + size : offset + 12 + size])[0]
|
|
assert zlib.crc32(payload) == crc
|
|
offset += size + 12
|
|
pngs.append({"file": str(p.relative_to(root)), "sha256": hashlib.sha256(raw).hexdigest()})
|
|
assert sum(series.values()) == 135 and len(capacities) == 15 and len(pngs) == 12
|
|
all_rows = [row for rows in groups.values() for row in rows]
|
|
report = {
|
|
"date": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"commit": sha,
|
|
"accepted_series_counts": series,
|
|
"total_measurements": len(all_rows),
|
|
"triplicate_groups": len(groups),
|
|
"capacity_rows_recomputed": len(capacities),
|
|
"raw_k6_metrics_match": True,
|
|
"actual_resource_limits_match": True,
|
|
"disturbance_rules_pass": True,
|
|
"database_mb_range": [
|
|
min(float(r["db_mb"]) for r in all_rows),
|
|
max(float(r["db_mb"]) for r in all_rows),
|
|
],
|
|
"measured_requests": sum(int(r["requests"]) for r in all_rows),
|
|
"images_by_series_variant": {"/".join(k): sorted(v) for k, v in images.items()},
|
|
"png_integrity": pngs,
|
|
"pass": True,
|
|
}
|
|
(root / "evidence/final-validation/existing-evidence-audit.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{k: v for k, v in report.items() if k not in ["png_integrity", "images_by_series_variant"]},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|