#!/usr/bin/env python3 import argparse import datetime as dt import json import os import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SQL = ( "SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256," " floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint," " EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id)," " (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL" " FROM attachments a ORDER BY a.object_key" ) HASH = ( 'while read -r k; do printf "%s %s\\n" "$k" ' '"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done' ) CLASSES = ( "orphan_stale", "orphan_fresh", "missing_referenced", "missing_unreferenced", "size_mismatch", "sha_mismatch", "pending_stale_with_object", "pending_stale_without_object", "pending_fresh", "quarantined", "incomplete_stale", "incomplete_fresh", "lost_ready", ) TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"} BROKEN = ( "missing_referenced", "missing_unreferenced", "size_mismatch", "sha_mismatch", "quarantined", ) def timed(cmd, stdin=None): started = time.monotonic() out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) if out.returncode: raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}") return out.stdout, round(time.monotonic() - started, 3) def s3(*args, stdin=None): return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin) def age(stamp, now): return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp() def load(bucket, prefix, deep): listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}") uploads, uploads_seconds = s3( "object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}" ) user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"] rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL]) objects = {} for item in json.loads(listing).get("items", []): key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"] objects[key] = item rows = {} for line in rows_raw.splitlines(): rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t") rows[key] = { "id": rid, "status": status, "size": int(size), "sha256": sha, "age": int(row_age), "referenced": referenced == "t", "quarantined": quarantined == "t", } hashes, deep_seconds = {}, 0.0 if deep: present = "".join( f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects ) out, deep_seconds = s3("sh", "-c", HASH, stdin=present) hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line) seconds = { "s3_list": list_seconds, "s3_incomplete": uploads_seconds, "pg_query": pg_seconds, "deep_hash": deep_seconds, } return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds def row_class(key, row, objects, hashes, grace): if row["quarantined"]: return "quarantined" if row["status"] == "pending": stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object" return "pending_fresh" if row["age"] <= grace else stale if key not in objects: return "missing_referenced" if row["referenced"] else "missing_unreferenced" if objects[key]["size_bytes"] != row["size"]: return "size_mismatch" return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None def classify(objects, rows, hashes, uploads, grace): now = time.time() classes = {name: [] for name in CLASSES} for key, item in objects.items(): if key not in rows: classes[ "orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh" ].append(key) for key, row in rows.items(): name = row_class(key, row, objects, hashes, grace) if name: classes[name].append(key if name.startswith("orphan") else row["id"]) for item in uploads: classes[ "incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh" ].append(item["key"]) return classes def main(): p = argparse.ArgumentParser( description="Independent PG <-> S3 invariant check for lab4 attachments" ) p.add_argument( "--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600")) ) p.add_argument( "--deep", action="store_true", help="stream every present object and compare sha256" ) p.add_argument( "--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments" ) p.add_argument( "--expect-ready", metavar="FILE", help="fail if any attachment from FILE is no longer healthy", ) p.add_argument("--limit", type=int, default=20, help="max ids per class in the report") p.add_argument("--output", metavar="FILE") args = p.parse_args() started = time.monotonic() prefix = os.environ.get("S3_PREFIX", "attachments/") objects, rows, hashes, uploads, seconds = load( os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep ) classes = classify(objects, rows, hashes, uploads, args.grace) bad = {i for c in BROKEN for i in classes[c]} healthy = { r["id"]: r["sha256"] for k, r in rows.items() if r["status"] == "ready" and k in objects and r["id"] not in bad } if args.expect_ready: expected = json.loads(Path(args.expect_ready).read_text()) classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha) if args.save_ready: Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True)) violations = [c for c in CLASSES if classes[c] and c not in TOLERATED] seconds["total"] = round(time.monotonic() - started, 3) report = { "checked_at": dt.datetime.now(dt.timezone.utc) .isoformat(timespec="seconds") .replace("+00:00", "Z"), "grace_seconds": args.grace, "deep": args.deep, "objects": len(objects), "rows": len(rows), "ready_healthy": len(healthy), "counts": {c: len(v) for c, v in classes.items()}, "samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v}, "seconds": seconds, "violations": violations, "ok": not violations, } text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(text + "\n") print(text) sys.exit(0 if report["ok"] else 1) if __name__ == "__main__": main()