161 lines
5.3 KiB
Python
161 lines
5.3 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
root = Path(__file__).resolve().parents[3]
|
|
ev = root / "evidence/grade5"
|
|
|
|
|
|
def save(path, data):
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
|
|
|
|
|
def data(series):
|
|
return json.loads((ev / series / "analysis.json").read_text())
|
|
|
|
|
|
def decide_hi(series="improvement-01", hypothesis="HI"):
|
|
a = data(series)
|
|
pairs = []
|
|
for repeat in (1, 2, 3):
|
|
rows = {r["config"]: r for r in a["runs"] if r["repeat"] == repeat}
|
|
c, t = rows["I0"], rows["I1"]
|
|
valid = all(r["valid"] and r["correctness"]["passed"] for r in (c, t))
|
|
ratio = t["p95_ms"] / c["p95_ms"]
|
|
passed = (
|
|
valid
|
|
and ratio <= 0.85
|
|
and t["success_ratio"] >= max(0.99, c["success_ratio"])
|
|
and t["p95_ms"] <= 500
|
|
)
|
|
pairs.append(
|
|
{
|
|
"repeat": repeat,
|
|
"control_p95_ms": c["p95_ms"],
|
|
"tuned_p95_ms": t["p95_ms"],
|
|
"p95_reduction_percent": 100 * (1 - ratio),
|
|
"control_success_ratio": c["success_ratio"],
|
|
"tuned_success_ratio": t["success_ratio"],
|
|
"valid": valid,
|
|
"criteria_met": passed,
|
|
}
|
|
)
|
|
result = (
|
|
"supported"
|
|
if all(p["criteria_met"] for p in pairs)
|
|
else (
|
|
"refuted"
|
|
if all(p["valid"] and not p["criteria_met"] for p in pairs)
|
|
else "inconclusive"
|
|
)
|
|
)
|
|
save(
|
|
ev / series / "decision.json",
|
|
{"hypothesis": hypothesis, "result": result, "pairs": pairs},
|
|
)
|
|
|
|
|
|
def decide_hc(series="capacity-improvement-01"):
|
|
a = data(series)
|
|
assert len(a["runs"]) == 12
|
|
pairs = []
|
|
for repeat in (1, 2, 3):
|
|
caps = {r["config"]: r for r in a["capacities"] if r["repeat"] == repeat}
|
|
c, t = caps["I0"], caps["I1"]
|
|
rows = [r for r in a["runs"] if r["repeat"] == repeat]
|
|
valid = all(
|
|
r["valid"] and r["correctness"]["passed"] for r in rows
|
|
) and not any(r["nonmonotonic"] for r in (c, t))
|
|
lower = (
|
|
t["capacity_lower"] / c["capacity_upper"] if c["capacity_upper"] else None
|
|
)
|
|
upper = (
|
|
t["capacity_upper"] / c["capacity_lower"]
|
|
if t["capacity_upper"] and c["capacity_lower"]
|
|
else None
|
|
)
|
|
outcome = "inconclusive"
|
|
if valid and lower is not None and lower >= 1.15:
|
|
outcome = "supported"
|
|
elif valid and upper is not None and upper < 1.15:
|
|
outcome = "refuted"
|
|
pairs.append(
|
|
{
|
|
"repeat": repeat,
|
|
"control": c,
|
|
"tuned": t,
|
|
"ratio_lower_bound": lower,
|
|
"ratio_upper_bound": upper,
|
|
"valid": valid,
|
|
"decision": outcome,
|
|
}
|
|
)
|
|
outcomes = [p["decision"] for p in pairs]
|
|
result = outcomes[0] if len(set(outcomes)) == 1 else "inconclusive"
|
|
save(
|
|
ev / series / "decision.json",
|
|
{
|
|
"hypothesis": "HC",
|
|
"result": result,
|
|
"blocks": pairs,
|
|
"note": "Finite-grid bounds, not confidence intervals.",
|
|
},
|
|
)
|
|
|
|
|
|
def audit(series):
|
|
directory = ev / series
|
|
protocol = json.loads((directory / "protocol.json").read_text())
|
|
summaries = list(directory.glob("*/summary.json"))
|
|
expected = (
|
|
len(protocol["configs"]) * len(protocol["rates"]) * len(protocol["order_seeds"])
|
|
)
|
|
assert len(summaries) == expected
|
|
s = [json.loads(p.read_text()) for p in summaries]
|
|
protocol_sha = hashlib.sha256(
|
|
(directory / "protocol.json").read_bytes()
|
|
).hexdigest()
|
|
assert protocol_sha == (directory / "protocol.sha256").read_text().split()[0]
|
|
for p in summaries:
|
|
m = json.loads(p.with_name("manifest.json").read_text())
|
|
assert m["protocol_sha256"] == protocol_sha
|
|
assert m["code_sha256"] == protocol["code_sha256"]
|
|
assert m["instrument_sha256"] == protocol["instrument_sha256"]
|
|
save(
|
|
directory / "audit.json",
|
|
{
|
|
"planned_runs": expected,
|
|
"completed_runs": len(s),
|
|
"valid_runs": sum(r["valid"] for r in s),
|
|
"slo_passing_runs": sum(r["slo_pass"] for r in s),
|
|
"confirmed_writes_checked": sum(r["correctness"]["expected"] for r in s),
|
|
"all_confirmed_writes_preserved": all(
|
|
r["correctness"]["passed"] for r in s
|
|
),
|
|
"generator_dropped": sum(r["generator_dropped"] for r in s),
|
|
"oom_container_observations": sum(
|
|
c["oom"] for r in s for c in r["containers_after"]
|
|
),
|
|
"protocol_sha256": protocol_sha,
|
|
"source_archive_sha256": hashlib.sha256(
|
|
(directory / "source.tar.gz").read_bytes()
|
|
).hexdigest(),
|
|
},
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
"Audit saved runs and apply preregistered decision rules"
|
|
)
|
|
parser.add_argument("mode", choices=("audit", "hi", "hc", "hl"))
|
|
parser.add_argument("series")
|
|
args = parser.parse_args()
|
|
if args.mode in ("hi", "hl"):
|
|
decide_hi(args.series, args.mode.upper())
|
|
elif args.mode == "hc":
|
|
decide_hc(args.series)
|
|
audit(args.series)
|