#!/usr/bin/env python3 """Find S3 objects with no matching `attachments` row (e.g. PUT succeeded but the PostgreSQL INSERT that followed it never committed -- a crash between the two writes). Dry-run by default; --delete removes orphans older than the grace period, never anything referenced in the database. Usage: python3 scripts/reconcile_attachments.py --grace-minutes 60 python3 scripts/reconcile_attachments.py --grace-minutes 60 --delete """ import argparse import asyncio import datetime as dt import os import sys import asyncpg import boto3 def read_secret(file_env, inline_env): path = os.environ.get(file_env) if path and os.path.isfile(path): return open(path, encoding="utf-8").read().strip() return os.environ[inline_env] async def main(): p = argparse.ArgumentParser() p.add_argument("--grace-minutes", type=int, default=60, help="Only objects older than this are candidates (protects in-flight uploads).") p.add_argument("--delete", action="store_true", help="Actually delete orphans (default: dry-run).") args = p.parse_args() database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL") conn = await asyncpg.connect(database_url) known_keys = {r["s3_key"] for r in await conn.fetch("SELECT s3_key FROM attachments")} await conn.close() s3 = boto3.client( "s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.environ.get("S3_REGION", "us-east-1"), aws_access_key_id=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), aws_secret_access_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), ) bucket = os.environ.get("S3_BUCKET", "messenger-attachments") cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=args.grace_minutes) orphans = [] paginator = s3.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"): for obj in page.get("Contents", []): if obj["Key"] in known_keys: continue if obj["LastModified"] > cutoff: continue # might still be an in-flight upload; not an orphan yet orphans.append(obj["Key"]) print(f"{len(orphans)} orphaned object(s) older than {args.grace_minutes} minutes with no DB row.") for key in orphans: print(" ", key) if args.delete: s3.delete_object(Bucket=bucket, Key=key) if orphans and args.delete: print(f"Deleted {len(orphans)} orphan(s).") elif orphans: print("Dry-run: nothing deleted. Re-run with --delete to remove them.") if __name__ == "__main__": sys.exit(asyncio.run(main()) or 0)