forked from mtusi2026/mtusi2026lab4
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
175 lines
8.6 KiB
Python
175 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
|
|
|
|
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
|
|
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
|
|
unused ready row, same-size corruption (found only with --verify-checksums);
|
|
DB side: stale intents (with / without object), quarantine mark on a healthy row;
|
|
fresh: orphan + intent younger than --grace.
|
|
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
|
|
lost object from a "backup" -> apply (quarantine lifted).
|
|
Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them ->
|
|
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
|
|
third run finds nothing.
|
|
|
|
python bench/reconcile_faults.py --output evidence/reconcile
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "student-tests"))
|
|
import imagegen # noqa: E402
|
|
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
|
|
s3_put)
|
|
from test_reconcile import insert_intent, snapshot # noqa: E402
|
|
|
|
GRACE = ["--grace", "30", "--allow-small-grace"]
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
|
|
|
|
def reconcile(*args: str) -> tuple[int, dict]:
|
|
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
|
|
return r.returncode, json.loads(r.stdout)
|
|
|
|
|
|
def save(out: Path, name: str, data) -> None:
|
|
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
|
|
|
|
def ours(report: dict, keys: dict[str, str]) -> dict:
|
|
"""The findings of this scenario's keys only (the stand also holds other test data)."""
|
|
f = findings_by_key(report)
|
|
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
|
|
for label, k in keys.items()}
|
|
|
|
|
|
def scenario_a(out: Path) -> dict:
|
|
api = Api()
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
chat = api.chat(alice, bob)
|
|
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")}
|
|
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
|
|
for n, a in up.items()}
|
|
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
|
|
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
|
|
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
|
|
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
|
|
"same-size corruption F": key["F"]}
|
|
orphan = "o/" + uuid.uuid4().hex
|
|
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
|
|
s3_put(orphan, SAMPLE)
|
|
s3_put(stale_with, SAMPLE)
|
|
insert_intent(chat, alice, stale_with, SAMPLE, 600)
|
|
insert_intent(chat, alice, stale_without, SAMPLE, 600)
|
|
s3_delete(key["C"])
|
|
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
|
|
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
|
|
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
|
|
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
|
|
time.sleep(32)
|
|
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
|
|
s3_put(fresh_orphan, SAMPLE)
|
|
s3_put(fresh_intent, SAMPLE)
|
|
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
|
|
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
|
|
|
|
before = snapshot()
|
|
_, dry = reconcile("--verify-checksums", *GRACE)
|
|
after = snapshot()
|
|
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
|
|
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
|
|
s3_put(key["C"], data["C"]) # the lost object restored from a backup
|
|
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
|
|
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
|
|
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
|
|
save(out, name, report)
|
|
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
|
|
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
|
|
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
|
|
present = s3_keys()
|
|
summary = {
|
|
"dry_run_changed_nothing": before == after,
|
|
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
|
|
"after restore of C": ours(apply3, keys)},
|
|
"objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()},
|
|
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
|
|
"content_status_after": content,
|
|
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
|
|
apply3["duration_seconds"]],
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
|
|
return summary
|
|
|
|
|
|
def scenario_b(out: Path, count: int = 2000) -> dict:
|
|
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
|
|
s3_exec("ks = sys.stdin.read().split()\n"
|
|
"sem = asyncio.Semaphore(32)\n"
|
|
"async def one(k):\n"
|
|
" async with sem:\n"
|
|
" await s.put(k, b'orphan', 'image/png', {})\n"
|
|
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
|
|
time.sleep(32)
|
|
result: dict = {}
|
|
|
|
def run_apply() -> None:
|
|
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
|
|
|
|
t = threading.Thread(target=run_apply)
|
|
t.start()
|
|
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
left = len(s3_keys() & set(keys))
|
|
if left < count:
|
|
break
|
|
time.sleep(0.2)
|
|
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
|
|
t.join()
|
|
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
|
|
time.sleep(5) # let S3 settle after unpause before counting
|
|
left_after_interrupt = len(s3_keys() & set(keys))
|
|
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
|
|
code3, third = reconcile("--apply", *GRACE)
|
|
save(out, "interrupted-1-apply.json", result["report"])
|
|
save(out, "interrupted-2-rerun.json", second)
|
|
save(out, "interrupted-3-third.json", third)
|
|
summary = {
|
|
"orphans_injected": count,
|
|
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
|
|
"applied": result["report"]["actions"]["applied"]},
|
|
"orphans_left_after_interrupt": left_after_interrupt,
|
|
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
|
|
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
|
|
"orphans_left_at_end": len(s3_keys() & set(keys)),
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
|
|
return summary
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--output", default="evidence/reconcile")
|
|
args = p.parse_args()
|
|
out = ROOT / args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
a = scenario_a(out)
|
|
b = scenario_b(out)
|
|
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
|
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
|
|
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|