Files
mtusi2026lab3/experiments/lab3-scale/postprocess/decisions.py
T
2026-10-10 21:32:33 +03:00

161 lines
5.3 KiB
Python

import hashlib
import json
from pathlib import Path
root = Path(__file__).resolve().parents[3]
ev = root / "evidence/grade5"
def save(path, data):
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
def data(series):
return json.loads((ev / series / "analysis.json").read_text())
def decide_hi(series="improvement-01", hypothesis="HI"):
a = data(series)
pairs = []
for repeat in (1, 2, 3):
rows = {r["config"]: r for r in a["runs"] if r["repeat"] == repeat}
c, t = rows["I0"], rows["I1"]
valid = all(r["valid"] and r["correctness"]["passed"] for r in (c, t))
ratio = t["p95_ms"] / c["p95_ms"]
passed = (
valid
and ratio <= 0.85
and t["success_ratio"] >= max(0.99, c["success_ratio"])
and t["p95_ms"] <= 500
)
pairs.append(
{
"repeat": repeat,
"control_p95_ms": c["p95_ms"],
"tuned_p95_ms": t["p95_ms"],
"p95_reduction_percent": 100 * (1 - ratio),
"control_success_ratio": c["success_ratio"],
"tuned_success_ratio": t["success_ratio"],
"valid": valid,
"criteria_met": passed,
}
)
result = (
"supported"
if all(p["criteria_met"] for p in pairs)
else (
"refuted"
if all(p["valid"] and not p["criteria_met"] for p in pairs)
else "inconclusive"
)
)
save(
ev / series / "decision.json",
{"hypothesis": hypothesis, "result": result, "pairs": pairs},
)
def decide_hc(series="capacity-improvement-01"):
a = data(series)
assert len(a["runs"]) == 12
pairs = []
for repeat in (1, 2, 3):
caps = {r["config"]: r for r in a["capacities"] if r["repeat"] == repeat}
c, t = caps["I0"], caps["I1"]
rows = [r for r in a["runs"] if r["repeat"] == repeat]
valid = all(
r["valid"] and r["correctness"]["passed"] for r in rows
) and not any(r["nonmonotonic"] for r in (c, t))
lower = (
t["capacity_lower"] / c["capacity_upper"] if c["capacity_upper"] else None
)
upper = (
t["capacity_upper"] / c["capacity_lower"]
if t["capacity_upper"] and c["capacity_lower"]
else None
)
outcome = "inconclusive"
if valid and lower is not None and lower >= 1.15:
outcome = "supported"
elif valid and upper is not None and upper < 1.15:
outcome = "refuted"
pairs.append(
{
"repeat": repeat,
"control": c,
"tuned": t,
"ratio_lower_bound": lower,
"ratio_upper_bound": upper,
"valid": valid,
"decision": outcome,
}
)
outcomes = [p["decision"] for p in pairs]
result = outcomes[0] if len(set(outcomes)) == 1 else "inconclusive"
save(
ev / series / "decision.json",
{
"hypothesis": "HC",
"result": result,
"blocks": pairs,
"note": "Finite-grid bounds, not confidence intervals.",
},
)
def audit(series):
directory = ev / series
protocol = json.loads((directory / "protocol.json").read_text())
summaries = list(directory.glob("*/summary.json"))
expected = (
len(protocol["configs"]) * len(protocol["rates"]) * len(protocol["order_seeds"])
)
assert len(summaries) == expected
s = [json.loads(p.read_text()) for p in summaries]
protocol_sha = hashlib.sha256(
(directory / "protocol.json").read_bytes()
).hexdigest()
assert protocol_sha == (directory / "protocol.sha256").read_text().split()[0]
for p in summaries:
m = json.loads(p.with_name("manifest.json").read_text())
assert m["protocol_sha256"] == protocol_sha
assert m["code_sha256"] == protocol["code_sha256"]
assert m["instrument_sha256"] == protocol["instrument_sha256"]
save(
directory / "audit.json",
{
"planned_runs": expected,
"completed_runs": len(s),
"valid_runs": sum(r["valid"] for r in s),
"slo_passing_runs": sum(r["slo_pass"] for r in s),
"confirmed_writes_checked": sum(r["correctness"]["expected"] for r in s),
"all_confirmed_writes_preserved": all(
r["correctness"]["passed"] for r in s
),
"generator_dropped": sum(r["generator_dropped"] for r in s),
"oom_container_observations": sum(
c["oom"] for r in s for c in r["containers_after"]
),
"protocol_sha256": protocol_sha,
"source_archive_sha256": hashlib.sha256(
(directory / "source.tar.gz").read_bytes()
).hexdigest(),
},
)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(
"Audit saved runs and apply preregistered decision rules"
)
parser.add_argument("mode", choices=("audit", "hi", "hc", "hl"))
parser.add_argument("series")
args = parser.parse_args()
if args.mode in ("hi", "hl"):
decide_hi(args.series, args.mode.upper())
elif args.mode == "hc":
decide_hc(args.series)
audit(args.series)