reconcile-inject.sh and reconcile_check.py come from the prepared kit: the first breaks either side on purpose (an object without a row, a removed object, another size, the same size with other bytes, an old intent with and without its object), the second is an oracle outside the API that classifies every row and object from psql and rc, hashes objects with --deep and reports ready attachments lost against a saved set. reconcile_track.py demo uploads through the balancer, links some attachments to messages, injects each class on both sides with old and fresh variants, and checks that a deep dry-run finds exactly what was injected and changes nothing, that apply fixes exactly that and a second apply does nothing, that broken rows and their message links survive in quarantine with metadata 200 and content 404, that a restored object heals, that a second pass is skipped while the lock is held, and that passes in a loop during 75 s of uploads lose nothing. Intents are injected 300 s old, between the reconciler grace and the API sweep grace, so only the reconciler removes them. reconcile_track.py cost snapshots docker stats of the whole host, pins S3, PostgreSQL and the reconciler to their own CPUs, fills the stand to each N with seed_storage.py run in the API image (parallel PutObject and COPY), and records three dry runs per N and mode with a median table. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
200 lines
7.1 KiB
Python
Executable File
200 lines
7.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SQL = (
|
|
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
|
|
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
|
|
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
|
|
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
|
|
" FROM attachments a ORDER BY a.object_key"
|
|
)
|
|
HASH = (
|
|
'while read -r k; do printf "%s %s\\n" "$k" '
|
|
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
|
|
)
|
|
CLASSES = (
|
|
"orphan_stale",
|
|
"orphan_fresh",
|
|
"missing_referenced",
|
|
"missing_unreferenced",
|
|
"size_mismatch",
|
|
"sha_mismatch",
|
|
"pending_stale_with_object",
|
|
"pending_stale_without_object",
|
|
"pending_fresh",
|
|
"quarantined",
|
|
"incomplete_stale",
|
|
"incomplete_fresh",
|
|
"lost_ready",
|
|
)
|
|
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
|
|
BROKEN = (
|
|
"missing_referenced",
|
|
"missing_unreferenced",
|
|
"size_mismatch",
|
|
"sha_mismatch",
|
|
"quarantined",
|
|
)
|
|
|
|
|
|
def timed(cmd, stdin=None):
|
|
started = time.monotonic()
|
|
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
|
if out.returncode:
|
|
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
|
|
return out.stdout, round(time.monotonic() - started, 3)
|
|
|
|
|
|
def s3(*args, stdin=None):
|
|
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
|
|
|
|
|
|
def age(stamp, now):
|
|
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
|
|
|
|
|
|
def load(bucket, prefix, deep):
|
|
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
|
|
uploads, uploads_seconds = s3(
|
|
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
|
|
)
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
|
|
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
|
|
objects = {}
|
|
for item in json.loads(listing).get("items", []):
|
|
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
|
|
objects[key] = item
|
|
rows = {}
|
|
for line in rows_raw.splitlines():
|
|
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
|
|
rows[key] = {
|
|
"id": rid,
|
|
"status": status,
|
|
"size": int(size),
|
|
"sha256": sha,
|
|
"age": int(row_age),
|
|
"referenced": referenced == "t",
|
|
"quarantined": quarantined == "t",
|
|
}
|
|
hashes, deep_seconds = {}, 0.0
|
|
if deep:
|
|
present = "".join(
|
|
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
|
|
)
|
|
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
|
|
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
|
|
seconds = {
|
|
"s3_list": list_seconds,
|
|
"s3_incomplete": uploads_seconds,
|
|
"pg_query": pg_seconds,
|
|
"deep_hash": deep_seconds,
|
|
}
|
|
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
|
|
|
|
|
|
def row_class(key, row, objects, hashes, grace):
|
|
if row["quarantined"]:
|
|
return "quarantined"
|
|
if row["status"] == "pending":
|
|
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
|
|
return "pending_fresh" if row["age"] <= grace else stale
|
|
if key not in objects:
|
|
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
|
|
if objects[key]["size_bytes"] != row["size"]:
|
|
return "size_mismatch"
|
|
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
|
|
|
|
|
|
def classify(objects, rows, hashes, uploads, grace):
|
|
now = time.time()
|
|
classes = {name: [] for name in CLASSES}
|
|
for key, item in objects.items():
|
|
if key not in rows:
|
|
classes[
|
|
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
|
|
].append(key)
|
|
for key, row in rows.items():
|
|
name = row_class(key, row, objects, hashes, grace)
|
|
if name:
|
|
classes[name].append(key if name.startswith("orphan") else row["id"])
|
|
for item in uploads:
|
|
classes[
|
|
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
|
|
].append(item["key"])
|
|
return classes
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Independent PG <-> S3 invariant check for lab4 attachments"
|
|
)
|
|
p.add_argument(
|
|
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
|
|
)
|
|
p.add_argument(
|
|
"--deep", action="store_true", help="stream every present object and compare sha256"
|
|
)
|
|
p.add_argument(
|
|
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
|
|
)
|
|
p.add_argument(
|
|
"--expect-ready",
|
|
metavar="FILE",
|
|
help="fail if any attachment from FILE is no longer healthy",
|
|
)
|
|
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
started = time.monotonic()
|
|
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
|
objects, rows, hashes, uploads, seconds = load(
|
|
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
|
|
)
|
|
classes = classify(objects, rows, hashes, uploads, args.grace)
|
|
bad = {i for c in BROKEN for i in classes[c]}
|
|
healthy = {
|
|
r["id"]: r["sha256"]
|
|
for k, r in rows.items()
|
|
if r["status"] == "ready" and k in objects and r["id"] not in bad
|
|
}
|
|
if args.expect_ready:
|
|
expected = json.loads(Path(args.expect_ready).read_text())
|
|
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
|
|
if args.save_ready:
|
|
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
|
|
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
|
|
seconds["total"] = round(time.monotonic() - started, 3)
|
|
report = {
|
|
"checked_at": dt.datetime.now(dt.timezone.utc)
|
|
.isoformat(timespec="seconds")
|
|
.replace("+00:00", "Z"),
|
|
"grace_seconds": args.grace,
|
|
"deep": args.deep,
|
|
"objects": len(objects),
|
|
"rows": len(rows),
|
|
"ready_healthy": len(healthy),
|
|
"counts": {c: len(v) for c, v in classes.items()},
|
|
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
|
|
"seconds": seconds,
|
|
"violations": violations,
|
|
"ok": not violations,
|
|
}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
sys.exit(0 if report["ok"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|