Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.
Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
200 lines
7.1 KiB
Python
Executable File
200 lines
7.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SQL = (
|
|
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
|
|
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
|
|
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
|
|
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
|
|
" FROM attachments a ORDER BY a.object_key"
|
|
)
|
|
HASH = (
|
|
'while read -r k; do printf "%s %s\\n" "$k" '
|
|
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
|
|
)
|
|
CLASSES = (
|
|
"orphan_stale",
|
|
"orphan_fresh",
|
|
"missing_referenced",
|
|
"missing_unreferenced",
|
|
"size_mismatch",
|
|
"sha_mismatch",
|
|
"pending_stale_with_object",
|
|
"pending_stale_without_object",
|
|
"pending_fresh",
|
|
"quarantined",
|
|
"incomplete_stale",
|
|
"incomplete_fresh",
|
|
"lost_ready",
|
|
)
|
|
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
|
|
BROKEN = (
|
|
"missing_referenced",
|
|
"missing_unreferenced",
|
|
"size_mismatch",
|
|
"sha_mismatch",
|
|
"quarantined",
|
|
)
|
|
|
|
|
|
def timed(cmd, stdin=None):
|
|
started = time.monotonic()
|
|
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
|
if out.returncode:
|
|
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
|
|
return out.stdout, round(time.monotonic() - started, 3)
|
|
|
|
|
|
def s3(*args, stdin=None):
|
|
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
|
|
|
|
|
|
def age(stamp, now):
|
|
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
|
|
|
|
|
|
def load(bucket, prefix, deep):
|
|
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
|
|
uploads, uploads_seconds = s3(
|
|
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
|
|
)
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
|
|
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
|
|
objects = {}
|
|
for item in json.loads(listing).get("items", []):
|
|
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
|
|
objects[key] = item
|
|
rows = {}
|
|
for line in rows_raw.splitlines():
|
|
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
|
|
rows[key] = {
|
|
"id": rid,
|
|
"status": status,
|
|
"size": int(size),
|
|
"sha256": sha,
|
|
"age": int(row_age),
|
|
"referenced": referenced == "t",
|
|
"quarantined": quarantined == "t",
|
|
}
|
|
hashes, deep_seconds = {}, 0.0
|
|
if deep:
|
|
present = "".join(
|
|
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
|
|
)
|
|
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
|
|
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
|
|
seconds = {
|
|
"s3_list": list_seconds,
|
|
"s3_incomplete": uploads_seconds,
|
|
"pg_query": pg_seconds,
|
|
"deep_hash": deep_seconds,
|
|
}
|
|
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
|
|
|
|
|
|
def row_class(key, row, objects, hashes, grace):
|
|
if row["quarantined"]:
|
|
return "quarantined"
|
|
if row["status"] == "pending":
|
|
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
|
|
return "pending_fresh" if row["age"] <= grace else stale
|
|
if key not in objects:
|
|
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
|
|
if objects[key]["size_bytes"] != row["size"]:
|
|
return "size_mismatch"
|
|
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
|
|
|
|
|
|
def classify(objects, rows, hashes, uploads, grace):
|
|
now = time.time()
|
|
classes = {name: [] for name in CLASSES}
|
|
for key, item in objects.items():
|
|
if key not in rows:
|
|
classes[
|
|
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
|
|
].append(key)
|
|
for key, row in rows.items():
|
|
name = row_class(key, row, objects, hashes, grace)
|
|
if name:
|
|
classes[name].append(key if name.startswith("orphan") else row["id"])
|
|
for item in uploads:
|
|
classes[
|
|
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
|
|
].append(item["key"])
|
|
return classes
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Independent PG <-> S3 invariant check for lab4 attachments"
|
|
)
|
|
p.add_argument(
|
|
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
|
|
)
|
|
p.add_argument(
|
|
"--deep", action="store_true", help="stream every present object and compare sha256"
|
|
)
|
|
p.add_argument(
|
|
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
|
|
)
|
|
p.add_argument(
|
|
"--expect-ready",
|
|
metavar="FILE",
|
|
help="fail if any attachment from FILE is no longer healthy",
|
|
)
|
|
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
started = time.monotonic()
|
|
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
|
objects, rows, hashes, uploads, seconds = load(
|
|
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
|
|
)
|
|
classes = classify(objects, rows, hashes, uploads, args.grace)
|
|
bad = {i for c in BROKEN for i in classes[c]}
|
|
healthy = {
|
|
r["id"]: r["sha256"]
|
|
for k, r in rows.items()
|
|
if r["status"] == "ready" and k in objects and r["id"] not in bad
|
|
}
|
|
if args.expect_ready:
|
|
expected = json.loads(Path(args.expect_ready).read_text())
|
|
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
|
|
if args.save_ready:
|
|
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
|
|
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
|
|
seconds["total"] = round(time.monotonic() - started, 3)
|
|
report = {
|
|
"checked_at": dt.datetime.now(dt.timezone.utc)
|
|
.isoformat(timespec="seconds")
|
|
.replace("+00:00", "Z"),
|
|
"grace_seconds": args.grace,
|
|
"deep": args.deep,
|
|
"objects": len(objects),
|
|
"rows": len(rows),
|
|
"ready_healthy": len(healthy),
|
|
"counts": {c: len(v) for c, v in classes.items()},
|
|
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
|
|
"seconds": seconds,
|
|
"violations": violations,
|
|
"ok": not violations,
|
|
}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
sys.exit(0 if report["ok"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|