#!/usr/bin/env python3 """Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report. Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved): S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an unused ready row, same-size corruption (found only with --verify-checksums); DB side: stale intents (with / without object), quarantine mark on a healthy row; fresh: orphan + intent younger than --grace. Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the lost object from a "backup" -> apply (quarantine lifted). Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them -> the run stops with an error, nothing else is damaged; the next run finishes the rest; the third run finds nothing. python bench/reconcile_faults.py --output evidence/reconcile """ from __future__ import annotations import argparse import json import subprocess import sys import threading import time import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "student-tests")) import imagegen # noqa: E402 from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402 s3_put) from test_reconcile import insert_intent, snapshot # noqa: E402 GRACE = ["--grace", "30", "--allow-small-grace"] SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() def reconcile(*args: str) -> tuple[int, dict]: r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600) return r.returncode, json.loads(r.stdout) def save(out: Path, name: str, data) -> None: (out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8") def ours(report: dict, keys: dict[str, str]) -> dict: """The findings of this scenario's keys only (the stand also holds other test data).""" f = findings_by_key(report) return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None) for label, k in keys.items()} def scenario_a(out: Path) -> dict: api = Api() alice, bob = api.user("alice"), api.user("bob") chat = api.chat(alice, bob) up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")} data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content for n, a in up.items()} key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()} api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]}) keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"], "unused, object truncated D": key["D"], "quarantined but healthy E": key["E"], "same-size corruption F": key["F"]} orphan = "o/" + uuid.uuid4().hex stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex s3_put(orphan, SAMPLE) s3_put(stale_with, SAMPLE) insert_intent(chat, alice, stale_with, SAMPLE, 600) insert_intent(chat, alice, stale_without, SAMPLE, 600) s3_delete(key["C"]) s3_put(key["D"], data["D"][: len(data["D"]) // 2]) s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:]) psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'") keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without}) time.sleep(32) fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex s3_put(fresh_orphan, SAMPLE) s3_put(fresh_intent, SAMPLE) insert_intent(chat, alice, fresh_intent, SAMPLE, 0) keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent}) before = snapshot() _, dry = reconcile("--verify-checksums", *GRACE) after = snapshot() _, apply1 = reconcile("--apply", "--verify-checksums", *GRACE) _, apply2 = reconcile("--apply", "--verify-checksums", *GRACE) s3_put(key["C"], data["C"]) # the lost object restored from a backup _, apply3 = reconcile("--apply", "--verify-checksums", *GRACE) for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1), ("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)): save(out, name, report) rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))} content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content", headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"} present = s3_keys() summary = { "dry_run_changed_nothing": before == after, "per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys), "after restore of C": ours(apply3, keys)}, "objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()}, "rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()}, "content_status_after": content, "durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"], apply3["duration_seconds"]], } print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True) return summary def scenario_b(out: Path, count: int = 2000) -> dict: keys = ["o/" + uuid.uuid4().hex for _ in range(count)] s3_exec("ks = sys.stdin.read().split()\n" "sem = asyncio.Semaphore(32)\n" "async def one(k):\n" " async with sem:\n" " await s.put(k, b'orphan', 'image/png', {})\n" "await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys)) time.sleep(32) result: dict = {} def run_apply() -> None: result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) t = threading.Thread(target=run_apply) t.start() # pause S3 as soon as deletions are under way (the reconciler is planning/listing first) deadline = time.monotonic() + 60 while time.monotonic() < deadline: left = len(s3_keys() & set(keys)) if left < count: break time.sleep(0.2) subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True) t.join() subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True) time.sleep(5) # let S3 settle after unpause before counting left_after_interrupt = len(s3_keys() & set(keys)) code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) code3, third = reconcile("--apply", *GRACE) save(out, "interrupted-1-apply.json", result["report"]) save(out, "interrupted-2-rerun.json", second) save(out, "interrupted-3-third.json", third) summary = { "orphans_injected": count, "interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"), "applied": result["report"]["actions"]["applied"]}, "orphans_left_after_interrupt": left_after_interrupt, "rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]}, "third": {"exit_code": code3, "planned": third["actions"]["planned"]}, "orphans_left_at_end": len(s3_keys() & set(keys)), } print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True) return summary def main() -> int: p = argparse.ArgumentParser() p.add_argument("--output", default="evidence/reconcile") args = p.parse_args() out = ROOT / args.output out.mkdir(parents=True, exist_ok=True) a = scenario_a(out) b = scenario_b(out) commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit, "grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b}) return 0 if __name__ == "__main__": sys.exit(main())