Files
mtusi2026lab2/evidence/validation/audit.py
T

221 lines
8.4 KiB
Python

import importlib.util
import json
import statistics
import subprocess
from collections import Counter
from datetime import datetime, timezone
from hashlib import sha256
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
COST = ROOT / "evidence/forward-published/cost"
MEASURED = "bc1d7aa7713858acb5d65b61d07e69a640f1ee97"
REVIEWED = "6ca1c7549978af3127966540385195ab0587f2e1"
ENTRY = "1e00d861a803a6d49ef443175b8231cbf048fc02"
SKELETON = "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1"
def git(*args):
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
def read(path):
return json.loads(path.read_text())
def digest(path):
return sha256(path.read_bytes()).hexdigest()
def main():
changed = git("diff", "--name-only", MEASURED, "HEAD").splitlines()
runtime_changes = [
p
for p in changed
if not p.startswith(("evidence/", "openspec/")) and p not in {"AGENTS.md", "REPORT.md"}
]
assert not runtime_changes, runtime_changes
reviewed_changes = git("diff", "--name-only", REVIEWED, "HEAD").splitlines()
assert all(
p in {"AGENTS.md", "REPORT.md"} or p.startswith("evidence/") for p in reviewed_changes
)
protected = [
p
for p in git("ls-tree", "-r", "--name-only", SKELETON).splitlines()
if p.startswith(("contracts/", "tests/"))
or p in {"README.md", "COURSE.md", "infra/README.md", "infra/otel/README.md"}
]
assert not git("diff", "--name-only", SKELETON, "HEAD", "--", *protected)
preserved = read(ROOT / "evidence/forward-published/preserved-mechanisms.json")
assert not git(
"diff",
"--name-only",
preserved["base_sha"],
"HEAD",
"--",
*preserved["unchanged_telemetry_paths"],
)
spec = importlib.util.spec_from_file_location("summary", ROOT / "scripts/obs_cost_summary.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
comparison = read(COST / "comparison.json")
saved = {r["file"]: r for r in comparison["runs"]}
timing = {r["file"]: r for r in read(COST / "measurement-audit.json")["runs"]}
runtime = read(ROOT / "evidence/forward-published/runtime.json")
assert runtime["application_sha"] == MEASURED
limits = {c["name"]: c["memory"] for c in runtime["containers"]}
results = []
summaries = []
all_ids = set()
modes = ("off", "on", "head", "tail", "balanced")
for mode in modes:
for repeat in range(1, 4):
path = COST / (mode + "-" + str(repeat) + ".json")
run = read(path)
summary = module.summarize(path)
assert summary == saved[path.name], path.name
assert (run["sha"], run["mode"], run["repeat"], run["duration"], run["rate"]) == (
MEASURED,
mode,
repeat,
40,
20,
)
assert run["result"] == "PASS" and run["unexpected_responses"] == 0
assert Counter(r["category"] for r in run["requests"]) == {
"normal": 768,
"error": 16,
"slow": 16,
}
assert all(
r["status"] == (503 if r["category"] == "error" else 200) for r in run["requests"]
)
ids = {r["id"] for r in run["requests"]}
assert len(ids) == 800 and not ids & all_ids
all_ids.update(ids)
load = run["finished"] - run["started"]
drain = (
datetime.fromisoformat(run["utc"].replace("Z", "+00:00")).timestamp()
- run["finished"]
)
samples = run["resources"]
gap = max(b["at"] - a["at"] for a, b in zip(samples, samples[1:]))
assert 38 <= load <= 42 and 30 <= drain <= 45 and gap < 5
assert timing[path.name] == {
"file": path.name,
"load_seconds": load,
"finish_to_report_seconds": drain,
"max_resource_gap_seconds": gap,
"result": "usable",
}
assert len(samples) == 21
container_ids = {c["id"] for c in samples[0]["containers"]}
assert len(container_ids) == 14
for sample in samples:
assert {c["id"] for c in sample["containers"]} == container_ids
assert all(c["memory_limit"] == limits[c["name"]] for c in sample["containers"])
assert all(row.endswith(" mtusi2026lab6") for row in run["background"])
assert summary["sdk_ended_spans"] == summary["collector_spans"]["accepted"]
assert summary["collector_spans"]["failed"] == 0
if mode in {"on", "tail", "balanced"}:
assert all(summary["traces"][kind]["retained"] == 16 for kind in ("error", "slow"))
if mode == "on":
assert summary["traces"]["normal"]["retained"] == 768
log = path.with_suffix(".txt").read_text()
assert MEASURED in log and "exit_code: 0" in log
assert "--mode " + mode + " --repeat " + str(repeat) + " --duration 40 --rate 20" in log
summaries.append(summary)
results.append(
{
"file": str(path.relative_to(ROOT)),
"sha256": digest(path),
"requests": 800,
"load_seconds": load,
"drain_seconds": drain,
"max_resource_gap_seconds": gap,
"result": "accepted",
}
)
groups = {
mode: {
key: {
"median": statistics.median(r[key] for r in summaries if r["mode"] == mode),
"min": min(r[key] for r in summaries if r["mode"] == mode),
"max": max(r[key] for r in summaries if r["mode"] == mode),
}
for key in ("p50_ms", "p95_ms", "p99_ms", "normal_p95_ms")
}
for mode in modes
}
assert groups == comparison["latency_ranges"]
excluded = read(COST / "invalid-series.json")
for run in excluded["runs"]:
for name, expected in run["files"].items():
assert digest(COST / "invalid-time-gap" / name) == expected
conditions = []
for path in sorted(COST.glob("conditions-*.json")):
data = read(path)
assert data["runtime_sha"] == MEASURED and data["other_projects"] == {}
assert (data["host_cpus"], data["architecture"], data["own_project"]) == (
16,
"arm64",
"mtusi2026lab6",
)
conditions.append({"file": str(path.relative_to(ROOT)), "sha256": digest(path)})
medians = {
mode: statistics.median(
r["ingest"]["tempo"]["tempo_distributor_bytes_received_total"]
for r in summaries
if r["mode"] == mode
)
for mode in modes
}
output = {
"at": datetime.now(timezone.utc).isoformat(),
"entry_sha": ENTRY,
"measured_sha": MEASURED,
"reviewed_sha": REVIEWED,
"runtime_changes": runtime_changes,
"reviewed_changes": reviewed_changes,
"protected_files": len(protected),
"unchanged_telemetry_paths": len(preserved["unchanged_telemetry_paths"]),
"accepted": results,
"excluded_hashes_verified": 4,
"condition_snapshots": conditions,
"total_requests": len(all_ids),
"unexpected_responses": 0,
"comparison_matches_raw": True,
"balanced_tempo_reduction_percent": 100 * (1 - medians["balanced"] / medians["on"]),
"retained": {
mode: {
kind: sum(r["traces"][kind]["retained"] for r in summaries if r["mode"] == mode)
for kind in ("normal", "error", "slow")
}
for mode in modes
},
"new_load_runs": 0,
"missing_or_invalid_required_series": [],
"result": "PASS",
}
target = Path(__file__).with_name("evidence-audit.json")
target.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n")
print(
json.dumps(
{
k: output[k]
for k in (
"result",
"total_requests",
"balanced_tempo_reduction_percent",
"new_load_runs",
"protected_files",
)
},
ensure_ascii=False,
)
)
if __name__ == "__main__":
main()