forked from mtusi2026/mtusi2026lab4
71 lines
2.8 KiB
Python
71 lines
2.8 KiB
Python
"""Find and safely repair objects that disagree with attachment metadata."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import boto3
|
|
import psycopg
|
|
from botocore.config import Config
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--apply', action='store_true', help='delete old orphans and mark missing rows failed')
|
|
parser.add_argument('--grace-seconds', type=int, default=300)
|
|
args = parser.parse_args()
|
|
|
|
database_url = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
|
|
bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
|
|
s3 = boto3.client(
|
|
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://minio:9000'),
|
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
|
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'messenger-api'),
|
|
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123'),
|
|
config=Config(connect_timeout=2, read_timeout=5, retries={'max_attempts': 1}),
|
|
)
|
|
objects = {}
|
|
token = None
|
|
while True:
|
|
kwargs = {'Bucket': bucket}
|
|
if token:
|
|
kwargs['ContinuationToken'] = token
|
|
page = s3.list_objects_v2(**kwargs)
|
|
for item in page.get('Contents', []):
|
|
objects[item['Key']] = item
|
|
if not page.get('IsTruncated'):
|
|
break
|
|
token = page['NextContinuationToken']
|
|
|
|
with psycopg.connect(database_url) as connection:
|
|
rows = connection.execute('SELECT id, object_key, status FROM attachments').fetchall()
|
|
known = {row[1]: row for row in rows}
|
|
missing = [row for row in rows if row[1] not in objects and row[2] == 'ready']
|
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds)
|
|
orphan = [item for key, item in objects.items() if key not in known and item['LastModified'] < cutoff]
|
|
deleted = []
|
|
marked_failed = []
|
|
if args.apply:
|
|
for item in orphan:
|
|
s3.delete_object(Bucket=bucket, Key=item['Key'])
|
|
deleted.append(item['Key'])
|
|
for row in missing:
|
|
connection.execute(
|
|
"UPDATE attachments SET status='failed', error_code='object_missing', updated_at=now() WHERE id=%s",
|
|
(row[0],),
|
|
)
|
|
marked_failed.append(str(row[0]))
|
|
connection.commit()
|
|
|
|
print(json.dumps({
|
|
'objects': len(objects), 'metadata_rows': len(rows),
|
|
'orphan_candidates': [item['Key'] for item in orphan],
|
|
'missing_candidates': [str(row[0]) for row in missing],
|
|
'deleted_orphans': deleted, 'marked_failed': marked_failed,
|
|
'dry_run': not args.apply, 'grace_seconds': args.grace_seconds,
|
|
}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|