From 504f77cb7cef28202e1830fc04e10e646488e778 Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 03:32:06 +0300 Subject: [PATCH] Validate reconciler evidence and expose isolated demo output --- Makefile | 2 +- tools/summarize_reconciler.py | 111 ++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tools/summarize_reconciler.py diff --git a/Makefile b/Makefile index 3b303e01..d8781f03 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ reconcile: $(COMPOSE) run --rm --no-deps reconciler $(ARGS) reconcile-demo: - $(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" + $(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS) reconcile-cost: $(PYTHON) scripts/reconcile_track.py cost $(ARGS) diff --git a/tools/summarize_reconciler.py b/tools/summarize_reconciler.py new file mode 100644 index 00000000..a61c6305 --- /dev/null +++ b/tools/summarize_reconciler.py @@ -0,0 +1,111 @@ +import argparse +import json +import math +from pathlib import Path +from statistics import median + +SIZES = (1000, 10000, 30000, 100000) +REPEATS = 3 +LINEAR_BOUNDS = (7, 13) +RSS_BOUND = 1.1 +DEEP_ORDER = 10 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate and compare complete reconciler series") + parser.add_argument("current", type=Path) + parser.add_argument("baseline", type=Path) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + current = json.loads((args.current / "summary.json").read_text()) + baseline = json.loads(args.baseline.read_text()) + rows = {(r["objects"], r["mode"]): r for r in current["results"]} + old = {(r["objects"], r["mode"]): r for r in baseline["results"]} + expected = {(n, mode) for n in SIZES for mode in ("shallow", "deep")} | {(10000, "page100")} + if set(rows) != expected or any(r["runs"] != REPEATS for r in rows.values()): + parser.error("expected all nine points with three repeats") + comparisons = [] + for (n, mode), row in sorted(rows.items()): + runs = [] + for repeat in range(1, REPEATS + 1): + raw = json.loads((args.current / f"n{n}-{mode}-{repeat}.json").read_text()) + runs.append(raw) + pages = math.ceil(n / (100 if mode == "page100" else 1000)) + requests = { + "s3_list": pages, + "sql": pages, + "s3_head": 0, + "s3_delete": 0, + "s3_get": n if mode == "deep" else 0, + } + if ( + raw["cost"]["requests"] != requests + or raw["screen"]["disturbed"] + or raw["scanned"] != {"objects": n, "rows": n} + or raw["errors"] + or raw["exit_code"] + or raw["skipped"] + or any(raw["found"].values()) + or raw["commit"] != current["commit"] + ): + parser.error(f"invalid accepted measurement: {n}/{mode}/{repeat}") + times = row["seconds_all"] + if ( + times != [r["cost"]["seconds"] for r in runs] + or row["seconds_median"] != median(times) + or row["max_rss_mib_median"] != median(r["cost"]["max_rss_mib"] for r in runs) + ): + parser.error(f"summary differs from raw measurements: {n}/{mode}") + previous = old[(n, mode)] + comparisons.append( + { + **row, + "seconds_range": [min(times), max(times)], + "spread_percent": round((max(times) - min(times)) / row["seconds_median"] * 100, 2), + "historical_seconds": previous["seconds_median"], + "historical_rss_mib": previous["max_rss_mib_median"], + "time_ratio_to_historical": row["seconds_median"] / previous["seconds_median"], + "rss_ratio_to_historical": row["max_rss_mib_median"] + / previous["max_rss_mib_median"], + } + ) + ratio = rows[(100000, "shallow")]["seconds_median"] / rows[(10000, "shallow")]["seconds_median"] + rss = ( + rows[(100000, "shallow")]["max_rss_mib_median"] + / rows[(1000, "shallow")]["max_rss_mib_median"] + ) + deep = { + str(n): rows[(n, "deep")]["seconds_median"] / rows[(n, "shallow")]["seconds_median"] + for n in SIZES + } + result = { + "current_commit": current["commit"], + "baseline_commit": baseline["commit"], + "accepted": len(rows) * REPEATS, + "rejected_files": len(list((args.current / "disturbed").glob("*.json"))), + "requests_and_consistency_verified_for_every_raw_run": True, + "h2": { + "t100k_over_t10k": ratio, + "within_7_to_13": LINEAR_BOUNDS[0] <= ratio <= LINEAR_BOUNDS[1], + "rss100k_over_rss1k": rss, + "rss_growth_below_10_percent": rss < RSS_BOUND, + "deep_over_shallow": deep, + "deep_at_least_tenfold": all(v >= DEEP_ORDER for v in deep.values()), + }, + "comparison": comparisons, + "limitation": ( + "Historical comparison is not an isolated causal A/B experiment; " + "warm seeded data in Docker Desktop." + ), + } + with args.out.open("x") as output: + output.write(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print( + json.dumps( + {k: v for k, v in result.items() if k != "comparison"}, ensure_ascii=False, indent=2 + ) + ) + + +if __name__ == "__main__": + main()