#!/usr/bin/env python3 """Safe S3/PostgreSQL attachment inventory. Dry-run unless --apply is specified.""" import argparse import datetime as dt import json import os from pathlib import Path import sys import time import boto3 from botocore.config import Config import psycopg from psycopg.rows import dict_row sys.path.insert(0,str(Path(__file__).resolve().parents[1])) from app.migrate import setting def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('--apply',action='store_true',help='delete only unreferenced S3 objects older than the grace period') p.add_argument('--grace-seconds',type=int,default=86400) p.add_argument('--allow-short-grace',action='store_true',help='test-only override; use only with a dedicated --prefix') p.add_argument('--prefix',default='',help='restrict the scan and repair to one S3 prefix') a=p.parse_args() if a.grace_seconds<3600 and not (a.allow_short_grace and a.prefix): p.error('grace period must be at least 3600 seconds unless test-only override and a prefix are both supplied') endpoint=os.getenv('S3_ENDPOINT','http://rustfs:9000'); bucket=os.getenv('S3_BUCKET','messenger-attachments') started=time.monotonic() key=setting('S3_ACCESS_KEY_ID'); secret=setting('S3_SECRET_ACCESS_KEY') s3=boto3.client('s3',endpoint_url=endpoint,region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=key,aws_secret_access_key=secret,config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2})) with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row,connect_timeout=3) as db: if a.prefix: rows=db.execute('SELECT id,object_key,created_at FROM attachments WHERE object_key LIKE %s',(a.prefix.replace('\\','\\\\').replace('%','\\%').replace('_','\\_')+'%',)).fetchall() else: rows=db.execute('SELECT id,object_key,created_at FROM attachments').fetchall() by_key={r['object_key']:r for r in rows} objects=[]; token=None while True: args={'Bucket':bucket,'MaxKeys':1000} if a.prefix: args['Prefix']=a.prefix if token: args['ContinuationToken']=token page=s3.list_objects_v2(**args) objects.extend(page.get('Contents',[])) if not page.get('IsTruncated'): break token=page['NextContinuationToken'] object_keys={o['Key'] for o in objects} missing=sorted(k for k in by_key if k not in object_keys) now=dt.datetime.now(dt.timezone.utc); cutoff=now-dt.timedelta(seconds=a.grace_seconds) orphan=[o for o in objects if o['Key'] not in by_key] eligible=[o for o in orphan if o['LastModified']