diff --git a/bench/reconcile_faults.py b/bench/reconcile_faults.py index 4c41812..7cb11d4 100644 --- a/bench/reconcile_faults.py +++ b/bench/reconcile_faults.py @@ -8,7 +8,7 @@ Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but th fresh: orphan + intent younger than --grace. Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the lost object from a "backup" -> apply (quarantine lifted). -Scenario B (interrupted apply): 400 old orphans, S3 is paused while --apply deletes them -> +Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them -> the run stops with an error, nothing else is damaged; the next run finishes the rest; the third run finds nothing. @@ -56,7 +56,7 @@ def scenario_a(out: Path) -> dict: api = Api() alice, bob = api.user("alice"), api.user("bob") chat = api.chat(alice, bob) - up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30)).json() for i, n in enumerate("ABCDEF")} + up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")} data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content for n, a in up.items()} key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()} @@ -100,7 +100,7 @@ def scenario_a(out: Path) -> dict: "dry_run_changed_nothing": before == after, "per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys), "after restore of C": ours(apply3, keys)}, - "objects_present_after": {label: k in present for label, k in keys.items()}, + "objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()}, "rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()}, "content_status_after": content, "durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"], @@ -110,7 +110,7 @@ def scenario_a(out: Path) -> dict: return summary -def scenario_b(out: Path, count: int = 400) -> dict: +def scenario_b(out: Path, count: int = 2000) -> dict: keys = ["o/" + uuid.uuid4().hex for _ in range(count)] s3_exec("ks = sys.stdin.read().split()\n" "sem = asyncio.Semaphore(32)\n" @@ -136,7 +136,7 @@ def scenario_b(out: Path, count: int = 400) -> dict: subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True) t.join() subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True) - time.sleep(2) + time.sleep(5) # let S3 settle after unpause before counting left_after_interrupt = len(s3_keys() & set(keys)) code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) code3, third = reconcile("--apply", *GRACE)