Files
mtusi2026lab4/bench/reconcile_faults.py
T

175 lines
8.6 KiB
Python

#!/usr/bin/env python3
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
unused ready row, same-size corruption (found only with --verify-checksums);
DB side: stale intents (with / without object), quarantine mark on a healthy row;
fresh: orphan + intent younger than --grace.
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
lost object from a "backup" -> apply (quarantine lifted).
Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them ->
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
third run finds nothing.
python bench/reconcile_faults.py --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
s3_put)
from test_reconcile import insert_intent, snapshot # noqa: E402
GRACE = ["--grace", "30", "--allow-small-grace"]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def reconcile(*args: str) -> tuple[int, dict]:
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
return r.returncode, json.loads(r.stdout)
def save(out: Path, name: str, data) -> None:
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
def ours(report: dict, keys: dict[str, str]) -> dict:
"""The findings of this scenario's keys only (the stand also holds other test data)."""
f = findings_by_key(report)
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
for label, k in keys.items()}
def scenario_a(out: Path) -> dict:
api = Api()
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")}
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for n, a in up.items()}
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
"same-size corruption F": key["F"]}
orphan = "o/" + uuid.uuid4().hex
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(orphan, SAMPLE)
s3_put(stale_with, SAMPLE)
insert_intent(chat, alice, stale_with, SAMPLE, 600)
insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"])
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
time.sleep(32)
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
before = snapshot()
_, dry = reconcile("--verify-checksums", *GRACE)
after = snapshot()
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
s3_put(key["C"], data["C"]) # the lost object restored from a backup
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
save(out, name, report)
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
present = s3_keys()
summary = {
"dry_run_changed_nothing": before == after,
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
"after restore of C": ours(apply3, keys)},
"objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()},
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
"content_status_after": content,
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
apply3["duration_seconds"]],
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def scenario_b(out: Path, count: int = 2000) -> dict:
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
s3_exec("ks = sys.stdin.read().split()\n"
"sem = asyncio.Semaphore(32)\n"
"async def one(k):\n"
" async with sem:\n"
" await s.put(k, b'orphan', 'image/png', {})\n"
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
time.sleep(32)
result: dict = {}
def run_apply() -> None:
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
t = threading.Thread(target=run_apply)
t.start()
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
left = len(s3_keys() & set(keys))
if left < count:
break
time.sleep(0.2)
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
t.join()
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
time.sleep(5) # let S3 settle after unpause before counting
left_after_interrupt = len(s3_keys() & set(keys))
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
code3, third = reconcile("--apply", *GRACE)
save(out, "interrupted-1-apply.json", result["report"])
save(out, "interrupted-2-rerun.json", second)
save(out, "interrupted-3-third.json", third)
summary = {
"orphans_injected": count,
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
"applied": result["report"]["actions"]["applied"]},
"orphans_left_after_interrupt": left_after_interrupt,
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
"orphans_left_at_end": len(s3_keys() & set(keys)),
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
a = scenario_a(out)
b = scenario_b(out)
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
return 0
if __name__ == "__main__":
sys.exit(main())