forked from mtusi2026/mtusi2026lab2
221 lines
8.4 KiB
Python
221 lines
8.4 KiB
Python
import importlib.util
|
|
import json
|
|
import statistics
|
|
import subprocess
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
COST = ROOT / "evidence/forward-published/cost"
|
|
MEASURED = "bc1d7aa7713858acb5d65b61d07e69a640f1ee97"
|
|
REVIEWED = "6ca1c7549978af3127966540385195ab0587f2e1"
|
|
ENTRY = "1e00d861a803a6d49ef443175b8231cbf048fc02"
|
|
SKELETON = "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1"
|
|
|
|
|
|
def git(*args):
|
|
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
|
|
|
|
|
|
def read(path):
|
|
return json.loads(path.read_text())
|
|
|
|
|
|
def digest(path):
|
|
return sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def main():
|
|
changed = git("diff", "--name-only", MEASURED, "HEAD").splitlines()
|
|
runtime_changes = [
|
|
p
|
|
for p in changed
|
|
if not p.startswith(("evidence/", "openspec/")) and p not in {"AGENTS.md", "REPORT.md"}
|
|
]
|
|
assert not runtime_changes, runtime_changes
|
|
reviewed_changes = git("diff", "--name-only", REVIEWED, "HEAD").splitlines()
|
|
assert all(
|
|
p in {"AGENTS.md", "REPORT.md"} or p.startswith("evidence/") for p in reviewed_changes
|
|
)
|
|
protected = [
|
|
p
|
|
for p in git("ls-tree", "-r", "--name-only", SKELETON).splitlines()
|
|
if p.startswith(("contracts/", "tests/"))
|
|
or p in {"README.md", "COURSE.md", "infra/README.md", "infra/otel/README.md"}
|
|
]
|
|
assert not git("diff", "--name-only", SKELETON, "HEAD", "--", *protected)
|
|
preserved = read(ROOT / "evidence/forward-published/preserved-mechanisms.json")
|
|
assert not git(
|
|
"diff",
|
|
"--name-only",
|
|
preserved["base_sha"],
|
|
"HEAD",
|
|
"--",
|
|
*preserved["unchanged_telemetry_paths"],
|
|
)
|
|
spec = importlib.util.spec_from_file_location("summary", ROOT / "scripts/obs_cost_summary.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
comparison = read(COST / "comparison.json")
|
|
saved = {r["file"]: r for r in comparison["runs"]}
|
|
timing = {r["file"]: r for r in read(COST / "measurement-audit.json")["runs"]}
|
|
runtime = read(ROOT / "evidence/forward-published/runtime.json")
|
|
assert runtime["application_sha"] == MEASURED
|
|
limits = {c["name"]: c["memory"] for c in runtime["containers"]}
|
|
results = []
|
|
summaries = []
|
|
all_ids = set()
|
|
modes = ("off", "on", "head", "tail", "balanced")
|
|
for mode in modes:
|
|
for repeat in range(1, 4):
|
|
path = COST / (mode + "-" + str(repeat) + ".json")
|
|
run = read(path)
|
|
summary = module.summarize(path)
|
|
assert summary == saved[path.name], path.name
|
|
assert (run["sha"], run["mode"], run["repeat"], run["duration"], run["rate"]) == (
|
|
MEASURED,
|
|
mode,
|
|
repeat,
|
|
40,
|
|
20,
|
|
)
|
|
assert run["result"] == "PASS" and run["unexpected_responses"] == 0
|
|
assert Counter(r["category"] for r in run["requests"]) == {
|
|
"normal": 768,
|
|
"error": 16,
|
|
"slow": 16,
|
|
}
|
|
assert all(
|
|
r["status"] == (503 if r["category"] == "error" else 200) for r in run["requests"]
|
|
)
|
|
ids = {r["id"] for r in run["requests"]}
|
|
assert len(ids) == 800 and not ids & all_ids
|
|
all_ids.update(ids)
|
|
load = run["finished"] - run["started"]
|
|
drain = (
|
|
datetime.fromisoformat(run["utc"].replace("Z", "+00:00")).timestamp()
|
|
- run["finished"]
|
|
)
|
|
samples = run["resources"]
|
|
gap = max(b["at"] - a["at"] for a, b in zip(samples, samples[1:]))
|
|
assert 38 <= load <= 42 and 30 <= drain <= 45 and gap < 5
|
|
assert timing[path.name] == {
|
|
"file": path.name,
|
|
"load_seconds": load,
|
|
"finish_to_report_seconds": drain,
|
|
"max_resource_gap_seconds": gap,
|
|
"result": "usable",
|
|
}
|
|
assert len(samples) == 21
|
|
container_ids = {c["id"] for c in samples[0]["containers"]}
|
|
assert len(container_ids) == 14
|
|
for sample in samples:
|
|
assert {c["id"] for c in sample["containers"]} == container_ids
|
|
assert all(c["memory_limit"] == limits[c["name"]] for c in sample["containers"])
|
|
assert all(row.endswith(" mtusi2026lab6") for row in run["background"])
|
|
assert summary["sdk_ended_spans"] == summary["collector_spans"]["accepted"]
|
|
assert summary["collector_spans"]["failed"] == 0
|
|
if mode in {"on", "tail", "balanced"}:
|
|
assert all(summary["traces"][kind]["retained"] == 16 for kind in ("error", "slow"))
|
|
if mode == "on":
|
|
assert summary["traces"]["normal"]["retained"] == 768
|
|
log = path.with_suffix(".txt").read_text()
|
|
assert MEASURED in log and "exit_code: 0" in log
|
|
assert "--mode " + mode + " --repeat " + str(repeat) + " --duration 40 --rate 20" in log
|
|
summaries.append(summary)
|
|
results.append(
|
|
{
|
|
"file": str(path.relative_to(ROOT)),
|
|
"sha256": digest(path),
|
|
"requests": 800,
|
|
"load_seconds": load,
|
|
"drain_seconds": drain,
|
|
"max_resource_gap_seconds": gap,
|
|
"result": "accepted",
|
|
}
|
|
)
|
|
groups = {
|
|
mode: {
|
|
key: {
|
|
"median": statistics.median(r[key] for r in summaries if r["mode"] == mode),
|
|
"min": min(r[key] for r in summaries if r["mode"] == mode),
|
|
"max": max(r[key] for r in summaries if r["mode"] == mode),
|
|
}
|
|
for key in ("p50_ms", "p95_ms", "p99_ms", "normal_p95_ms")
|
|
}
|
|
for mode in modes
|
|
}
|
|
assert groups == comparison["latency_ranges"]
|
|
excluded = read(COST / "invalid-series.json")
|
|
for run in excluded["runs"]:
|
|
for name, expected in run["files"].items():
|
|
assert digest(COST / "invalid-time-gap" / name) == expected
|
|
conditions = []
|
|
for path in sorted(COST.glob("conditions-*.json")):
|
|
data = read(path)
|
|
assert data["runtime_sha"] == MEASURED and data["other_projects"] == {}
|
|
assert (data["host_cpus"], data["architecture"], data["own_project"]) == (
|
|
16,
|
|
"arm64",
|
|
"mtusi2026lab6",
|
|
)
|
|
conditions.append({"file": str(path.relative_to(ROOT)), "sha256": digest(path)})
|
|
medians = {
|
|
mode: statistics.median(
|
|
r["ingest"]["tempo"]["tempo_distributor_bytes_received_total"]
|
|
for r in summaries
|
|
if r["mode"] == mode
|
|
)
|
|
for mode in modes
|
|
}
|
|
output = {
|
|
"at": datetime.now(timezone.utc).isoformat(),
|
|
"entry_sha": ENTRY,
|
|
"measured_sha": MEASURED,
|
|
"reviewed_sha": REVIEWED,
|
|
"runtime_changes": runtime_changes,
|
|
"reviewed_changes": reviewed_changes,
|
|
"protected_files": len(protected),
|
|
"unchanged_telemetry_paths": len(preserved["unchanged_telemetry_paths"]),
|
|
"accepted": results,
|
|
"excluded_hashes_verified": 4,
|
|
"condition_snapshots": conditions,
|
|
"total_requests": len(all_ids),
|
|
"unexpected_responses": 0,
|
|
"comparison_matches_raw": True,
|
|
"balanced_tempo_reduction_percent": 100 * (1 - medians["balanced"] / medians["on"]),
|
|
"retained": {
|
|
mode: {
|
|
kind: sum(r["traces"][kind]["retained"] for r in summaries if r["mode"] == mode)
|
|
for kind in ("normal", "error", "slow")
|
|
}
|
|
for mode in modes
|
|
},
|
|
"new_load_runs": 0,
|
|
"missing_or_invalid_required_series": [],
|
|
"result": "PASS",
|
|
}
|
|
target = Path(__file__).with_name("evidence-audit.json")
|
|
target.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
k: output[k]
|
|
for k in (
|
|
"result",
|
|
"total_requests",
|
|
"balanced_tempo_reduction_percent",
|
|
"new_load_runs",
|
|
"protected_files",
|
|
)
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|