Files
mtusi2026lab4/scripts/reconcile.py
T
2026-11-06 17:03:03 +03:00

60 lines
3.4 KiB
Python
Executable File

#!/usr/bin/env python3
"""Safe S3/PostgreSQL attachment inventory. Dry-run unless --apply is specified."""
import argparse
import datetime as dt
import json
import os
from pathlib import Path
import sys
import time
import boto3
from botocore.config import Config
import psycopg
from psycopg.rows import dict_row
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from app.migrate import setting
def main():
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--apply',action='store_true',help='delete only unreferenced S3 objects older than the grace period')
p.add_argument('--grace-seconds',type=int,default=86400)
p.add_argument('--allow-short-grace',action='store_true',help='test-only override; use only with a dedicated --prefix')
p.add_argument('--prefix',default='',help='restrict the scan and repair to one S3 prefix')
a=p.parse_args()
if a.grace_seconds<3600 and not (a.allow_short_grace and a.prefix): p.error('grace period must be at least 3600 seconds unless test-only override and a prefix are both supplied')
endpoint=os.getenv('S3_ENDPOINT','http://rustfs:9000'); bucket=os.getenv('S3_BUCKET','messenger-attachments')
started=time.monotonic()
key=setting('S3_ACCESS_KEY_ID'); secret=setting('S3_SECRET_ACCESS_KEY')
s3=boto3.client('s3',endpoint_url=endpoint,region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=key,aws_secret_access_key=secret,config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2}))
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row,connect_timeout=3) as db:
if a.prefix:
rows=db.execute('SELECT id,object_key,created_at FROM attachments WHERE object_key LIKE %s',(a.prefix.replace('\\','\\\\').replace('%','\\%').replace('_','\\_')+'%',)).fetchall()
else:
rows=db.execute('SELECT id,object_key,created_at FROM attachments').fetchall()
by_key={r['object_key']:r for r in rows}
objects=[]; token=None
while True:
args={'Bucket':bucket,'MaxKeys':1000}
if a.prefix: args['Prefix']=a.prefix
if token: args['ContinuationToken']=token
page=s3.list_objects_v2(**args)
objects.extend(page.get('Contents',[]))
if not page.get('IsTruncated'): break
token=page['NextContinuationToken']
object_keys={o['Key'] for o in objects}
missing=sorted(k for k in by_key if k not in object_keys)
now=dt.datetime.now(dt.timezone.utc); cutoff=now-dt.timedelta(seconds=a.grace_seconds)
orphan=[o for o in objects if o['Key'] not in by_key]
eligible=[o for o in orphan if o['LastModified']<cutoff]
deleted=[]
if a.apply:
for o in eligible:
# Recheck the reference immediately before deletion to reduce the race with an upload/commit.
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as db:
exists=db.execute('SELECT 1 FROM attachments WHERE object_key=%s',(o['Key'],)).fetchone()
if exists: continue
s3.delete_object(Bucket=bucket,Key=o['Key']); deleted.append(o['Key'])
report={'mode':'apply' if a.apply else 'dry-run','bucket':bucket,'prefix':a.prefix or None,'db_rows':len(rows),'s3_objects':len(objects),'missing_objects':missing,'orphan_objects':[o['Key'] for o in orphan],'eligible_orphans':[o['Key'] for o in eligible],'deleted_objects':deleted,'grace_seconds':a.grace_seconds,'elapsed_seconds':round(time.monotonic()-started,4),'safe_policy':'missing referenced objects are reported only; only unreferenced objects older than grace can be deleted'}
print(json.dumps(report,indent=2))
if __name__=='__main__': main()