forked from mtusi2026/mtusi2026lab2
Validate reconciler evidence and expose isolated demo output
This commit is contained in:
@@ -170,7 +170,7 @@ reconcile:
|
||||
$(COMPOSE) run --rm --no-deps reconciler $(ARGS)
|
||||
|
||||
reconcile-demo:
|
||||
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)"
|
||||
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS)
|
||||
|
||||
reconcile-cost:
|
||||
$(PYTHON) scripts/reconcile_track.py cost $(ARGS)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
|
||||
SIZES = (1000, 10000, 30000, 100000)
|
||||
REPEATS = 3
|
||||
LINEAR_BOUNDS = (7, 13)
|
||||
RSS_BOUND = 1.1
|
||||
DEEP_ORDER = 10
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Validate and compare complete reconciler series")
|
||||
parser.add_argument("current", type=Path)
|
||||
parser.add_argument("baseline", type=Path)
|
||||
parser.add_argument("--out", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
current = json.loads((args.current / "summary.json").read_text())
|
||||
baseline = json.loads(args.baseline.read_text())
|
||||
rows = {(r["objects"], r["mode"]): r for r in current["results"]}
|
||||
old = {(r["objects"], r["mode"]): r for r in baseline["results"]}
|
||||
expected = {(n, mode) for n in SIZES for mode in ("shallow", "deep")} | {(10000, "page100")}
|
||||
if set(rows) != expected or any(r["runs"] != REPEATS for r in rows.values()):
|
||||
parser.error("expected all nine points with three repeats")
|
||||
comparisons = []
|
||||
for (n, mode), row in sorted(rows.items()):
|
||||
runs = []
|
||||
for repeat in range(1, REPEATS + 1):
|
||||
raw = json.loads((args.current / f"n{n}-{mode}-{repeat}.json").read_text())
|
||||
runs.append(raw)
|
||||
pages = math.ceil(n / (100 if mode == "page100" else 1000))
|
||||
requests = {
|
||||
"s3_list": pages,
|
||||
"sql": pages,
|
||||
"s3_head": 0,
|
||||
"s3_delete": 0,
|
||||
"s3_get": n if mode == "deep" else 0,
|
||||
}
|
||||
if (
|
||||
raw["cost"]["requests"] != requests
|
||||
or raw["screen"]["disturbed"]
|
||||
or raw["scanned"] != {"objects": n, "rows": n}
|
||||
or raw["errors"]
|
||||
or raw["exit_code"]
|
||||
or raw["skipped"]
|
||||
or any(raw["found"].values())
|
||||
or raw["commit"] != current["commit"]
|
||||
):
|
||||
parser.error(f"invalid accepted measurement: {n}/{mode}/{repeat}")
|
||||
times = row["seconds_all"]
|
||||
if (
|
||||
times != [r["cost"]["seconds"] for r in runs]
|
||||
or row["seconds_median"] != median(times)
|
||||
or row["max_rss_mib_median"] != median(r["cost"]["max_rss_mib"] for r in runs)
|
||||
):
|
||||
parser.error(f"summary differs from raw measurements: {n}/{mode}")
|
||||
previous = old[(n, mode)]
|
||||
comparisons.append(
|
||||
{
|
||||
**row,
|
||||
"seconds_range": [min(times), max(times)],
|
||||
"spread_percent": round((max(times) - min(times)) / row["seconds_median"] * 100, 2),
|
||||
"historical_seconds": previous["seconds_median"],
|
||||
"historical_rss_mib": previous["max_rss_mib_median"],
|
||||
"time_ratio_to_historical": row["seconds_median"] / previous["seconds_median"],
|
||||
"rss_ratio_to_historical": row["max_rss_mib_median"]
|
||||
/ previous["max_rss_mib_median"],
|
||||
}
|
||||
)
|
||||
ratio = rows[(100000, "shallow")]["seconds_median"] / rows[(10000, "shallow")]["seconds_median"]
|
||||
rss = (
|
||||
rows[(100000, "shallow")]["max_rss_mib_median"]
|
||||
/ rows[(1000, "shallow")]["max_rss_mib_median"]
|
||||
)
|
||||
deep = {
|
||||
str(n): rows[(n, "deep")]["seconds_median"] / rows[(n, "shallow")]["seconds_median"]
|
||||
for n in SIZES
|
||||
}
|
||||
result = {
|
||||
"current_commit": current["commit"],
|
||||
"baseline_commit": baseline["commit"],
|
||||
"accepted": len(rows) * REPEATS,
|
||||
"rejected_files": len(list((args.current / "disturbed").glob("*.json"))),
|
||||
"requests_and_consistency_verified_for_every_raw_run": True,
|
||||
"h2": {
|
||||
"t100k_over_t10k": ratio,
|
||||
"within_7_to_13": LINEAR_BOUNDS[0] <= ratio <= LINEAR_BOUNDS[1],
|
||||
"rss100k_over_rss1k": rss,
|
||||
"rss_growth_below_10_percent": rss < RSS_BOUND,
|
||||
"deep_over_shallow": deep,
|
||||
"deep_at_least_tenfold": all(v >= DEEP_ORDER for v in deep.values()),
|
||||
},
|
||||
"comparison": comparisons,
|
||||
"limitation": (
|
||||
"Historical comparison is not an isolated causal A/B experiment; "
|
||||
"warm seeded data in Docker Desktop."
|
||||
),
|
||||
}
|
||||
with args.out.open("x") as output:
|
||||
output.write(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
json.dumps(
|
||||
{k: v for k, v in result.items() if k != "comparison"}, ensure_ascii=False, indent=2
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user