#!/usr/bin/env python3 import argparse import hashlib import json import logging import sys import time from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Optional if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from src.store import Store else: from .store import Store LOGGER = logging.getLogger("messenger.reconcile") def reconcile( store: Store, grace_seconds: int, apply_changes: bool, restore_dir: Optional[Path] = None, ): started = time.monotonic() metadata = store.reconciliation_snapshot() cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) found = set() orphans = [] for key, last_modified in store.objects.objects(): found.add(key) if key not in metadata and last_modified <= cutoff: orphans.append(key) missing = [ row for key, row in metadata.items() if key not in found and row.get("reconcile_type", "original") == "original" ] restore_candidates = [] if restore_dir is not None: root = restore_dir.resolve(strict=True) for row in missing: source = restore_dir / str(row["id"]) try: resolved = source.resolve(strict=True) if ( resolved.parent != root or not resolved.is_file() or resolved.stat().st_size != row["size_bytes"] ): continue content = resolved.read_bytes() except OSError: continue if ( len(content) == row["size_bytes"] and hashlib.sha256(content).hexdigest() == row["sha256"].strip() ): restore_candidates.append((row, content)) restored = 0 if apply_changes: for key in orphans: store.objects.delete(key) restorable_ids = set() for row, content in restore_candidates: store.objects.put(row["object_key"], content, row["content_type"]) store.clear_missing_object(row["id"]) restorable_ids.add(row["id"]) restored += 1 for row in missing: if row["id"] not in restorable_ids: store.record_missing_object(row["id"]) for key, row in metadata.items(): if key in found and row.get("reconcile_type", "original") == "original": store.clear_missing_object(row["id"]) unresolved_missing = len(missing) - restored return { "mode": "apply" if apply_changes else "dry-run", "grace_seconds": grace_seconds, "objects_seen": len(found), "metadata_seen": len({str(row["id"]) for row in metadata.values()}), "orphan_objects": len([key for key in found if key not in metadata]), "stale_orphan_candidates": len(orphans), "deleted_orphans": len(orphans) if apply_changes else 0, "missing_objects": len(missing), "unresolved_missing_objects": unresolved_missing, "restorable_missing_objects": len(restore_candidates), "restored_missing_objects": restored, "missing_incidents_recorded": len(missing) - restored if apply_changes else 0, "duration_ms": round((time.monotonic() - started) * 1000, 3), } def main(): parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.") parser.add_argument("--grace-seconds", type=int, default=3600) parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects") parser.add_argument( "--restore-dir", type=Path, help="optional trusted backup directory with exact original bytes named by attachment UUID", ) args = parser.parse_args() if args.grace_seconds < 60: parser.error("--grace-seconds must be at least 60") logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") store = Store() store.open() try: result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir) print(json.dumps(result, sort_keys=True)) if result["unresolved_missing_objects"]: LOGGER.error( "%s database records reference missing S3 objects; metadata was retained for manual recovery", result["unresolved_missing_objects"], ) finally: store.close() if __name__ == "__main__": main()