From 827b783be355be9fa60117dc3eda350af3dab454 Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 02:29:53 +0300 Subject: [PATCH] Reject disturbed reconciler measurements and preserve every series --- .../tasks.md | 4 +- scripts/capture_evidence.py | 1 + scripts/reconcile_track.py | 77 +++++++++++++------ .../unit/test_reconcile_measurements.py | 69 +++++++++++++++++ 4 files changed, 124 insertions(+), 27 deletions(-) create mode 100644 student-tests/unit/test_reconcile_measurements.py diff --git a/openspec/changes/verify-lab4-reconciler-sqlalchemy/tasks.md b/openspec/changes/verify-lab4-reconciler-sqlalchemy/tasks.md index 6a8205db..8dd21a64 100644 --- a/openspec/changes/verify-lab4-reconciler-sqlalchemy/tasks.md +++ b/openspec/changes/verify-lab4-reconciler-sqlalchemy/tasks.md @@ -1,7 +1,7 @@ ## 1. Достоверность драйвера -- [ ] 1.1 Добавить новые каталоги, полный provenance, отказ от disturbed и checkpoint. -- [ ] 1.2 Проверить отказные пути драйвера, lint/check; закоммитить до опытов. +- [x] 1.1 Добавить новые каталоги, полный provenance, отказ от disturbed и checkpoint. +- [x] 1.2 Проверить отказные пути драйвера, lint/check; закоммитить до опытов. ## 2. Корректность diff --git a/scripts/capture_evidence.py b/scripts/capture_evidence.py index 6f597032..6f9ff851 100644 --- a/scripts/capture_evidence.py +++ b/scripts/capture_evidence.py @@ -15,6 +15,7 @@ SETTINGS = ( "LB_STATS_PORT", "BASE_URL", "EVIDENCE_DIR", + "RECONCILE_EVIDENCE_DIR", "NO_PROXY", "S3_CPUS", "S3_MEMORY", diff --git a/scripts/reconcile_track.py b/scripts/reconcile_track.py index 7731e8ab..87d3f30c 100644 --- a/scripts/reconcile_track.py +++ b/scripts/reconcile_track.py @@ -19,7 +19,6 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) client = __import__("client") FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() -OUT = ROOT / "evidence" / "reconcile" GRACE = 60 FLAGS = ("--grace", str(GRACE), "--skew", "0") INTENT_AGE = 300 @@ -99,13 +98,18 @@ def inject(kind, count, ids=(), age=None): def head(): - return must(["git", "rev-parse", "--short", "HEAD"]).strip() + return must(["git", "rev-parse", "HEAD"]).strip() -def save(name, data): - path = OUT / name +def evidence_root(): + return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile") + + +def save(name, data, *, stamp=True): + path = evidence_root() / name path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n") + stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data} + path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n") return path @@ -153,10 +157,10 @@ def faults(c, check, pre): "bitflip": [linked[3], ids[9], ids[10]], } damaged = {i for group in targets.values() for i in group} - ready = OUT / "ready-before.json" + ready = evidence_root() / "ready-before.json" oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready)) untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged} - save("ready-untouched.json", untouched) + save("ready-untouched.json", untouched, stamp=False) started = time.monotonic() injected = { "orphan_stale": inject("orphan", 3), @@ -171,7 +175,9 @@ def faults(c, check, pre): injected["pending_fresh"] = inject("stale-pending", 2, age=0) expected = {name: len(keys) for name, keys in injected.items()} expected.update(quarantined=pre["counts"]["quarantined"]) - before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json")) + before = oracle( + "--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json") + ) dry = reconcile("--dry-run", "--deep", *FLAGS) save("dry-run.json", dry) after_dry = oracle("--grace", str(GRACE), "--deep") @@ -216,9 +222,9 @@ def faults(c, check, pre): str(GRACE), "--deep", "--expect-ready", - str(OUT / "ready-untouched.json"), + str(evidence_root() / "ready-untouched.json"), "--output", - str(OUT / "check-after.json"), + str(evidence_root() / "check-after.json"), ) tolerated = ("orphan_fresh", "pending_fresh") check( @@ -406,7 +412,7 @@ def uploader(base, token, chat, *, deadline, results, lock): def under_load(c, check, base, seconds, threads): alice, _, _, chat = c.bootstrap() - ready = OUT / "ready-load.json" + ready = evidence_root() / "ready-load.json" left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"] leftover = { "orphan_stale": left["orphan_fresh"] + left["orphan_stale"], @@ -436,7 +442,7 @@ def under_load(c, check, base, seconds, threads): runs.append(reconcile("--apply", *FLAGS)) created = {att["id"]: att["sha256"] for status, att in results if att} expected = dict(json.loads(ready.read_text()), **created) - save("ready-load.json", expected) + save("ready-load.json", expected, stamp=False) after = oracle( "--grace", str(GRACE), @@ -444,7 +450,7 @@ def under_load(c, check, base, seconds, threads): "--expect-ready", str(ready), "--output", - str(OUT / "check-under-load.json"), + str(evidence_root() / "check-under-load.json"), ) statuses = {} for status, _ in results: @@ -495,7 +501,7 @@ def under_load(c, check, base, seconds, threads): def demo(args): - OUT.mkdir(parents=True, exist_ok=True) + evidence_root().mkdir(parents=True, exist_ok=False) c = client.Client(args.base_url, timeout=30) check = Checks() pre = oracle("--grace", str(GRACE)) @@ -608,19 +614,21 @@ def wait_quiet(args, pinned): quiet = screen(first, sample(), 0.0, pinned, args, started=False) calm = 0 if quiet["disturbed"] else calm + 1 sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n") + return calm == 3 def measure(flags, env, args, pinned): rejected = [] - for _ in range(args.retries + 1): + for attempt in range(args.retries + 1): first = sample() report = reconcile(*flags, "--samples", "3", env=env) report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args) if not report["screen"]["disturbed"]: - break + return report, rejected rejected.append(report) - wait_quiet(args, pinned) - return report, rejected + if attempt < args.retries and not wait_quiet(args, pinned): + break + return None, rejected def seed(count, env): @@ -709,13 +717,12 @@ def table(rows): def cost(args): - out = OUT / "cost" - (out / "disturbed").mkdir(parents=True, exist_ok=True) - for stale in [*out.glob("n*-*.json"), *(out / "disturbed").glob("*.json")]: - stale.unlink() + out = evidence_root() / "cost" + out.mkdir(parents=True, exist_ok=False) + (out / "disturbed").mkdir() rows = int(psql("SELECT count(*) FROM attachments")) if rows > args.sizes[0]: - raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test") + raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project") snapshot = out / "host-docker-stats.txt" snapshot.write_text(host("before the series")) env = pins(args) @@ -735,7 +742,9 @@ def cost(args): plan["seeds"].append(dict(seed(n - rows, env), target=n)) rows = n time.sleep(args.settle) - wait_quiet(args, pinned) + save("cost/plan.json", plan) + if not wait_quiet(args, pinned): + raise SystemExit("no quiet measurement window before timeout") modes = [("shallow", ["--dry-run"])] if n <= args.deep_max: modes.append(("deep", ["--dry-run", "--deep"])) @@ -748,10 +757,23 @@ def cost(args): for number, bad in enumerate(dropped, 1): save(f"cost/disturbed/{name}-try{number}.json", bad) rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped) - (out / f"{name}.json").write_text(json.dumps(report, indent=2) + "\n") + if report is None: + raise SystemExit(f"{name}: no clean run after retries; rejected preserved") + if ( + report["exit_code"] + or report["errors"] + or report["skipped"] + or report["scanned"] != {"objects": n, "rows": n} + or any(report["found"].values()) + ): + save(f"cost/invalid/{name}.json", report) + raise SystemExit(f"{name}: inconsistent dataset or failed pass") + save(f"cost/{name}.json", report) results.setdefault((n, mode), []).append(report) + save("cost/summary.json", {**plan, "results": summarize(results, rejected)}) sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n") finally: + save("cost/summary.json", {**plan, "results": summarize(results, rejected)}) compose("up", "-d", "--wait", "--no-deps", "s3", "postgres") with snapshot.open("a") as handle: handle.write("\n" + host("after the series")) @@ -784,7 +806,12 @@ def main(): k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores") k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run") k.add_argument("--quiet-timeout", type=float, default=900.0) + for command in (d, k): + command.add_argument( + "--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile") + ) args = p.parse_args() + os.environ["RECONCILE_EVIDENCE_DIR"] = args.out if args.command == "demo": demo(args) else: diff --git a/student-tests/unit/test_reconcile_measurements.py b/student-tests/unit/test_reconcile_measurements.py new file mode 100644 index 00000000..46257821 --- /dev/null +++ b/student-tests/unit/test_reconcile_measurements.py @@ -0,0 +1,69 @@ +import importlib.util +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def track(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + script = Path(__file__).resolve().parents[2] / "scripts/reconcile_track.py" + spec = importlib.util.spec_from_file_location("reconcile_measurements", script) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("disturbed", "quiet", "accepted", "discarded"), + [ + ([False], True, True, 0), + ([True, False], True, True, 1), + ([True, True], True, False, 2), + ([True], False, False, 1), + ], +) +def test_only_clean_measurements_are_accepted( + track: ModuleType, + monkeypatch: pytest.MonkeyPatch, + *, + disturbed: list[bool], + quiet: bool, + accepted: bool, + discarded: int, +) -> None: + reports = [{"cost": {"cpu_seconds": 1}, "attempt": i} for i in range(len(disturbed))] + monkeypatch.setattr(track, "sample", lambda: None) + monkeypatch.setattr(track, "reconcile", Mock(side_effect=reports)) + monkeypatch.setattr(track, "screen", Mock(side_effect=[{"disturbed": d} for d in disturbed])) + monkeypatch.setattr(track, "wait_quiet", lambda *_: quiet) + chosen, rejected = track.measure([], {}, SimpleNamespace(retries=1), set()) + assert (chosen is not None) == accepted + assert len(rejected) == discarded + assert all(r["screen"]["disturbed"] for r in rejected) + if chosen is not None: + assert not chosen["screen"]["disturbed"] + assert chosen not in rejected + + +def test_existing_series_is_preserved_before_docker_access( + track: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("RECONCILE_EVIDENCE_DIR", str(tmp_path)) + folder = tmp_path / "cost" + folder.mkdir() + raw = folder / "n1000-shallow-1.json" + raw.write_text("previous evidence") + query = Mock(side_effect=AssertionError("must not access the stand")) + monkeypatch.setattr(track, "psql", query) + with pytest.raises(FileExistsError): + track.cost(SimpleNamespace(sizes=[1000])) + assert raw.read_text() == "previous evidence" + query.assert_not_called() + + +def test_quiet_deadline_is_not_success(track: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(track.time, "monotonic", lambda: 10) + assert not track.wait_quiet(SimpleNamespace(quiet_timeout=0), set())