Validate reconciler evidence and expose isolated demo output

This commit is contained in:
Artyom
2026-11-06 03:32:06 +03:00
parent 3402fd69d4
commit 504f77cb7c
2 changed files with 112 additions and 1 deletions
+1 -1
View File
@@ -170,7 +170,7 @@ reconcile:
$(COMPOSE) run --rm --no-deps reconciler $(ARGS)
reconcile-demo:
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)"
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS)
reconcile-cost:
$(PYTHON) scripts/reconcile_track.py cost $(ARGS)
+111
View File
@@ -0,0 +1,111 @@
import argparse
import json
import math
from pathlib import Path
from statistics import median
SIZES = (1000, 10000, 30000, 100000)
REPEATS = 3
LINEAR_BOUNDS = (7, 13)
RSS_BOUND = 1.1
DEEP_ORDER = 10
def main() -> None:
parser = argparse.ArgumentParser(description="Validate and compare complete reconciler series")
parser.add_argument("current", type=Path)
parser.add_argument("baseline", type=Path)
parser.add_argument("--out", required=True, type=Path)
args = parser.parse_args()
current = json.loads((args.current / "summary.json").read_text())
baseline = json.loads(args.baseline.read_text())
rows = {(r["objects"], r["mode"]): r for r in current["results"]}
old = {(r["objects"], r["mode"]): r for r in baseline["results"]}
expected = {(n, mode) for n in SIZES for mode in ("shallow", "deep")} | {(10000, "page100")}
if set(rows) != expected or any(r["runs"] != REPEATS for r in rows.values()):
parser.error("expected all nine points with three repeats")
comparisons = []
for (n, mode), row in sorted(rows.items()):
runs = []
for repeat in range(1, REPEATS + 1):
raw = json.loads((args.current / f"n{n}-{mode}-{repeat}.json").read_text())
runs.append(raw)
pages = math.ceil(n / (100 if mode == "page100" else 1000))
requests = {
"s3_list": pages,
"sql": pages,
"s3_head": 0,
"s3_delete": 0,
"s3_get": n if mode == "deep" else 0,
}
if (
raw["cost"]["requests"] != requests
or raw["screen"]["disturbed"]
or raw["scanned"] != {"objects": n, "rows": n}
or raw["errors"]
or raw["exit_code"]
or raw["skipped"]
or any(raw["found"].values())
or raw["commit"] != current["commit"]
):
parser.error(f"invalid accepted measurement: {n}/{mode}/{repeat}")
times = row["seconds_all"]
if (
times != [r["cost"]["seconds"] for r in runs]
or row["seconds_median"] != median(times)
or row["max_rss_mib_median"] != median(r["cost"]["max_rss_mib"] for r in runs)
):
parser.error(f"summary differs from raw measurements: {n}/{mode}")
previous = old[(n, mode)]
comparisons.append(
{
**row,
"seconds_range": [min(times), max(times)],
"spread_percent": round((max(times) - min(times)) / row["seconds_median"] * 100, 2),
"historical_seconds": previous["seconds_median"],
"historical_rss_mib": previous["max_rss_mib_median"],
"time_ratio_to_historical": row["seconds_median"] / previous["seconds_median"],
"rss_ratio_to_historical": row["max_rss_mib_median"]
/ previous["max_rss_mib_median"],
}
)
ratio = rows[(100000, "shallow")]["seconds_median"] / rows[(10000, "shallow")]["seconds_median"]
rss = (
rows[(100000, "shallow")]["max_rss_mib_median"]
/ rows[(1000, "shallow")]["max_rss_mib_median"]
)
deep = {
str(n): rows[(n, "deep")]["seconds_median"] / rows[(n, "shallow")]["seconds_median"]
for n in SIZES
}
result = {
"current_commit": current["commit"],
"baseline_commit": baseline["commit"],
"accepted": len(rows) * REPEATS,
"rejected_files": len(list((args.current / "disturbed").glob("*.json"))),
"requests_and_consistency_verified_for_every_raw_run": True,
"h2": {
"t100k_over_t10k": ratio,
"within_7_to_13": LINEAR_BOUNDS[0] <= ratio <= LINEAR_BOUNDS[1],
"rss100k_over_rss1k": rss,
"rss_growth_below_10_percent": rss < RSS_BOUND,
"deep_over_shallow": deep,
"deep_at_least_tenfold": all(v >= DEEP_ORDER for v in deep.values()),
},
"comparison": comparisons,
"limitation": (
"Historical comparison is not an isolated causal A/B experiment; "
"warm seeded data in Docker Desktop."
),
}
with args.out.open("x") as output:
output.write(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(
json.dumps(
{k: v for k, v in result.items() if k != "comparison"}, ensure_ascii=False, indent=2
)
)
if __name__ == "__main__":
main()