60 lines
3.4 KiB
Python
Executable File
60 lines
3.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Safe S3/PostgreSQL attachment inventory. Dry-run unless --apply is specified."""
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import boto3
|
|
from botocore.config import Config
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
|
from app.migrate import setting
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--apply',action='store_true',help='delete only unreferenced S3 objects older than the grace period')
|
|
p.add_argument('--grace-seconds',type=int,default=86400)
|
|
p.add_argument('--allow-short-grace',action='store_true',help='test-only override; use only with a dedicated --prefix')
|
|
p.add_argument('--prefix',default='',help='restrict the scan and repair to one S3 prefix')
|
|
a=p.parse_args()
|
|
if a.grace_seconds<3600 and not (a.allow_short_grace and a.prefix): p.error('grace period must be at least 3600 seconds unless test-only override and a prefix are both supplied')
|
|
endpoint=os.getenv('S3_ENDPOINT','http://rustfs:9000'); bucket=os.getenv('S3_BUCKET','messenger-attachments')
|
|
started=time.monotonic()
|
|
key=setting('S3_ACCESS_KEY_ID'); secret=setting('S3_SECRET_ACCESS_KEY')
|
|
s3=boto3.client('s3',endpoint_url=endpoint,region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=key,aws_secret_access_key=secret,config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2}))
|
|
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row,connect_timeout=3) as db:
|
|
if a.prefix:
|
|
rows=db.execute('SELECT id,object_key,created_at FROM attachments WHERE object_key LIKE %s',(a.prefix.replace('\\','\\\\').replace('%','\\%').replace('_','\\_')+'%',)).fetchall()
|
|
else:
|
|
rows=db.execute('SELECT id,object_key,created_at FROM attachments').fetchall()
|
|
by_key={r['object_key']:r for r in rows}
|
|
objects=[]; token=None
|
|
while True:
|
|
args={'Bucket':bucket,'MaxKeys':1000}
|
|
if a.prefix: args['Prefix']=a.prefix
|
|
if token: args['ContinuationToken']=token
|
|
page=s3.list_objects_v2(**args)
|
|
objects.extend(page.get('Contents',[]))
|
|
if not page.get('IsTruncated'): break
|
|
token=page['NextContinuationToken']
|
|
object_keys={o['Key'] for o in objects}
|
|
missing=sorted(k for k in by_key if k not in object_keys)
|
|
now=dt.datetime.now(dt.timezone.utc); cutoff=now-dt.timedelta(seconds=a.grace_seconds)
|
|
orphan=[o for o in objects if o['Key'] not in by_key]
|
|
eligible=[o for o in orphan if o['LastModified']<cutoff]
|
|
deleted=[]
|
|
if a.apply:
|
|
for o in eligible:
|
|
# Recheck the reference immediately before deletion to reduce the race with an upload/commit.
|
|
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as db:
|
|
exists=db.execute('SELECT 1 FROM attachments WHERE object_key=%s',(o['Key'],)).fetchone()
|
|
if exists: continue
|
|
s3.delete_object(Bucket=bucket,Key=o['Key']); deleted.append(o['Key'])
|
|
report={'mode':'apply' if a.apply else 'dry-run','bucket':bucket,'prefix':a.prefix or None,'db_rows':len(rows),'s3_objects':len(objects),'missing_objects':missing,'orphan_objects':[o['Key'] for o in orphan],'eligible_orphans':[o['Key'] for o in eligible],'deleted_objects':deleted,'grace_seconds':a.grace_seconds,'elapsed_seconds':round(time.monotonic()-started,4),'safe_policy':'missing referenced objects are reported only; only unreferenced objects older than grace can be deleted'}
|
|
print(json.dumps(report,indent=2))
|
|
if __name__=='__main__': main()
|