Files
mtusi2026lab4/scripts/reconcile_s3.py
T

71 lines
2.8 KiB
Python

"""Find and safely repair objects that disagree with attachment metadata."""
import argparse
import json
import os
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
from botocore.config import Config
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--apply', action='store_true', help='delete old orphans and mark missing rows failed')
parser.add_argument('--grace-seconds', type=int, default=300)
args = parser.parse_args()
database_url = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
s3 = boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://minio:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'messenger-api'),
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123'),
config=Config(connect_timeout=2, read_timeout=5, retries={'max_attempts': 1}),
)
objects = {}
token = None
while True:
kwargs = {'Bucket': bucket}
if token:
kwargs['ContinuationToken'] = token
page = s3.list_objects_v2(**kwargs)
for item in page.get('Contents', []):
objects[item['Key']] = item
if not page.get('IsTruncated'):
break
token = page['NextContinuationToken']
with psycopg.connect(database_url) as connection:
rows = connection.execute('SELECT id, object_key, status FROM attachments').fetchall()
known = {row[1]: row for row in rows}
missing = [row for row in rows if row[1] not in objects and row[2] == 'ready']
cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds)
orphan = [item for key, item in objects.items() if key not in known and item['LastModified'] < cutoff]
deleted = []
marked_failed = []
if args.apply:
for item in orphan:
s3.delete_object(Bucket=bucket, Key=item['Key'])
deleted.append(item['Key'])
for row in missing:
connection.execute(
"UPDATE attachments SET status='failed', error_code='object_missing', updated_at=now() WHERE id=%s",
(row[0],),
)
marked_failed.append(str(row[0]))
connection.commit()
print(json.dumps({
'objects': len(objects), 'metadata_rows': len(rows),
'orphan_candidates': [item['Key'] for item in orphan],
'missing_candidates': [str(row[0]) for row in missing],
'deleted_orphans': deleted, 'marked_failed': marked_failed,
'dry_run': not args.apply, 'grace_seconds': args.grace_seconds,
}))
if __name__ == '__main__':
main()