Files
mtusi2026lab5/scripts/reconcile_check.py
T

200 lines
7.1 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import datetime as dt
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SQL = (
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
" FROM attachments a ORDER BY a.object_key"
)
HASH = (
'while read -r k; do printf "%s %s\\n" "$k" '
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
)
CLASSES = (
"orphan_stale",
"orphan_fresh",
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"pending_stale_with_object",
"pending_stale_without_object",
"pending_fresh",
"quarantined",
"incomplete_stale",
"incomplete_fresh",
"lost_ready",
)
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
BROKEN = (
"missing_referenced",
"missing_unreferenced",
"size_mismatch",
"sha_mismatch",
"quarantined",
)
def timed(cmd, stdin=None):
started = time.monotonic()
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
if out.returncode:
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
return out.stdout, round(time.monotonic() - started, 3)
def s3(*args, stdin=None):
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
def age(stamp, now):
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
def load(bucket, prefix, deep):
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
uploads, uploads_seconds = s3(
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
objects = {}
for item in json.loads(listing).get("items", []):
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
objects[key] = item
rows = {}
for line in rows_raw.splitlines():
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
rows[key] = {
"id": rid,
"status": status,
"size": int(size),
"sha256": sha,
"age": int(row_age),
"referenced": referenced == "t",
"quarantined": quarantined == "t",
}
hashes, deep_seconds = {}, 0.0
if deep:
present = "".join(
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
)
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
seconds = {
"s3_list": list_seconds,
"s3_incomplete": uploads_seconds,
"pg_query": pg_seconds,
"deep_hash": deep_seconds,
}
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
def row_class(key, row, objects, hashes, grace):
if row["quarantined"]:
return "quarantined"
if row["status"] == "pending":
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
return "pending_fresh" if row["age"] <= grace else stale
if key not in objects:
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
if objects[key]["size_bytes"] != row["size"]:
return "size_mismatch"
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
def classify(objects, rows, hashes, uploads, grace):
now = time.time()
classes = {name: [] for name in CLASSES}
for key, item in objects.items():
if key not in rows:
classes[
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
].append(key)
for key, row in rows.items():
name = row_class(key, row, objects, hashes, grace)
if name:
classes[name].append(key if name.startswith("orphan") else row["id"])
for item in uploads:
classes[
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
].append(item["key"])
return classes
def main():
p = argparse.ArgumentParser(
description="Independent PG <-> S3 invariant check for lab4 attachments"
)
p.add_argument(
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
)
p.add_argument(
"--deep", action="store_true", help="stream every present object and compare sha256"
)
p.add_argument(
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
)
p.add_argument(
"--expect-ready",
metavar="FILE",
help="fail if any attachment from FILE is no longer healthy",
)
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
started = time.monotonic()
prefix = os.environ.get("S3_PREFIX", "attachments/")
objects, rows, hashes, uploads, seconds = load(
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
)
classes = classify(objects, rows, hashes, uploads, args.grace)
bad = {i for c in BROKEN for i in classes[c]}
healthy = {
r["id"]: r["sha256"]
for k, r in rows.items()
if r["status"] == "ready" and k in objects and r["id"] not in bad
}
if args.expect_ready:
expected = json.loads(Path(args.expect_ready).read_text())
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
if args.save_ready:
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
seconds["total"] = round(time.monotonic() - started, 3)
report = {
"checked_at": dt.datetime.now(dt.timezone.utc)
.isoformat(timespec="seconds")
.replace("+00:00", "Z"),
"grace_seconds": args.grace,
"deep": args.deep,
"objects": len(objects),
"rows": len(rows),
"ready_healthy": len(healthy),
"counts": {c: len(v) for c, v in classes.items()},
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
"seconds": seconds,
"violations": violations,
"ok": not violations,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()