Files
mtusi2026lab2/scripts/reconcile_check.py
T
ArtyomandClaude Opus 5.5 a335f654ed Add the track 2 fault injection, invariant check and experiment driver
reconcile-inject.sh and reconcile_check.py come from the prepared kit:
the first breaks either side on purpose (an object without a row, a
removed object, another size, the same size with other bytes, an old
intent with and without its object), the second is an oracle outside
the API that classifies every row and object from psql and rc, hashes
objects with --deep and reports ready attachments lost against a saved
set.

reconcile_track.py demo uploads through the balancer, links some
attachments to messages, injects each class on both sides with old and
fresh variants, and checks that a deep dry-run finds exactly what was
injected and changes nothing, that apply fixes exactly that and a
second apply does nothing, that broken rows and their message links
survive in quarantine with metadata 200 and content 404, that a
restored object heals, that a second pass is skipped while the lock is
held, and that passes in a loop during 75 s of uploads lose nothing.
Intents are injected 300 s old, between the reconciler grace and the
API sweep grace, so only the reconciler removes them.

reconcile_track.py cost snapshots docker stats of the whole host, pins
S3, PostgreSQL and the reconciler to their own CPUs, fills the stand to
each N with seed_storage.py run in the API image (parallel PutObject and
COPY), and records three dry runs per N and mode with a median table.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-29 03:23:59 +03:00

200 lines
7.1 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SQL = (
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
" FROM attachments a ORDER BY a.object_key"
)
HASH = (
'while read -r k; do printf "%s %s\\n" "$k" '
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
)
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"pending_stale_with_object",
"pending_stale_without_object",
"pending_fresh",
"quarantined",
"incomplete_stale",
"incomplete_fresh",
"lost_ready",
)
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
BROKEN = (
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"quarantined",
)
def timed(cmd, stdin=None):
started = time.monotonic()
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
if out.returncode:
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
return out.stdout, round(time.monotonic() - started, 3)
def s3(*args, stdin=None):
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
def age(stamp, now):
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
def load(bucket, prefix, deep):
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
uploads, uploads_seconds = s3(
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
objects = {}
for item in json.loads(listing).get("items", []):
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
objects[key] = item
rows = {}
for line in rows_raw.splitlines():
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
rows[key] = {
"id": rid,
"status": status,
"size": int(size),
"sha256": sha,
"age": int(row_age),
"referenced": referenced == "t",
"quarantined": quarantined == "t",
}
hashes, deep_seconds = {}, 0.0
if deep:
present = "".join(
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
)
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
seconds = {
"s3_list": list_seconds,
"s3_incomplete": uploads_seconds,
"pg_query": pg_seconds,
"deep_hash": deep_seconds,
}
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
def row_class(key, row, objects, hashes, grace):
if row["quarantined"]:
return "quarantined"
if row["status"] == "pending":
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
return "pending_fresh" if row["age"] <= grace else stale
if key not in objects:
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
if objects[key]["size_bytes"] != row["size"]:
return "size_mismatch"
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
def classify(objects, rows, hashes, uploads, grace):
now = time.time()
classes = {name: [] for name in CLASSES}
for key, item in objects.items():
if key not in rows:
classes[
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
].append(key)
for key, row in rows.items():
name = row_class(key, row, objects, hashes, grace)
if name:
classes[name].append(key if name.startswith("orphan") else row["id"])
for item in uploads:
classes[
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
].append(item["key"])
return classes
def main():
p = argparse.ArgumentParser(
description="Independent PG <-> S3 invariant check for lab4 attachments"
)
p.add_argument(
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
)
p.add_argument(
"--deep", action="store_true", help="stream every present object and compare sha256"
)
p.add_argument(
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
)
p.add_argument(
"--expect-ready",
metavar="FILE",
help="fail if any attachment from FILE is no longer healthy",
)
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
started = time.monotonic()
prefix = os.environ.get("S3_PREFIX", "attachments/")
objects, rows, hashes, uploads, seconds = load(
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
)
classes = classify(objects, rows, hashes, uploads, args.grace)
bad = {i for c in BROKEN for i in classes[c]}
healthy = {
r["id"]: r["sha256"]
for k, r in rows.items()
if r["status"] == "ready" and k in objects and r["id"] not in bad
}
if args.expect_ready:
expected = json.loads(Path(args.expect_ready).read_text())
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
if args.save_ready:
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
seconds["total"] = round(time.monotonic() - started, 3)
report = {
"checked_at": dt.datetime.now(dt.timezone.utc)
.isoformat(timespec="seconds")
.replace("+00:00", "Z"),
"grace_seconds": args.grace,
"deep": args.deep,
"objects": len(objects),
"rows": len(rows),
"ready_healthy": len(healthy),
"counts": {c: len(v) for c, v in classes.items()},
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
"seconds": seconds,
"violations": violations,
"ok": not violations,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()