#!/usr/bin/env python3 """Dry-run first; remove only old, unreferenced objects or stale upload attempts.""" import argparse from datetime import datetime, timedelta, timezone import json from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from db import ApiError, Store from s3store import ObjectStore, S3Unavailable def parse_timestamp(value): return datetime.fromisoformat(value.replace('Z', '+00:00')) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--grace-seconds', type=int, default=900) parser.add_argument('--apply', action='store_true') args = parser.parse_args() if args.grace_seconds < 900: parser.error('--grace-seconds must be at least 900') store = Store.connect(run_migrations=False) objects = ObjectStore.from_env() try: cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) records = store.list_attachments_for_reconciliation() by_key = {record['object_key']: record for record in records} summary = { 'mode': 'apply' if args.apply else 'dry-run', 'grace_seconds': args.grace_seconds, 'stale_uploads': 0, 'old_failed_objects': 0, 'orphan_objects': 0, 'missing_ready_objects': 0, 'actions_applied': 0, } for record in records: key = record['object_key'] if (record['status'] == 'processing' and parse_timestamp(record['created_at']) < cutoff): summary['stale_uploads'] += 1 exists = objects.object_exists(key) if args.apply: if exists: objects.delete_object(key) store.mark_attachment_failed( record['id'], 'upload_interrupted', datetime.now(timezone.utc) ) summary['actions_applied'] += 1 elif (record['status'] == 'failed' and parse_timestamp(record['updated_at']) < cutoff): if objects.object_exists(key): summary['old_failed_objects'] += 1 if args.apply: objects.delete_object(key) summary['actions_applied'] += 1 elif record['status'] == 'ready' and not objects.object_exists(key): summary['missing_ready_objects'] += 1 if args.apply: store.mark_reconciled_missing( record['id'], datetime.now(timezone.utc) ) summary['actions_applied'] += 1 for item in objects.list_objects(): key = item['Key'] if key in by_key or item['LastModified'] >= cutoff: continue summary['orphan_objects'] += 1 if args.apply: objects.delete_object(key) summary['actions_applied'] += 1 print(json.dumps(summary, sort_keys=True)) finally: store.close() objects.client.close() if __name__ == '__main__': try: main() except (ApiError, S3Unavailable) as error: print('Reconciliation failed: ' + error.message if isinstance(error, ApiError) else 'Reconciliation failed: S3 is unavailable', file=sys.stderr) raise SystemExit(1) from None