forked from mtusi2026/mtusi2026lab4
92 lines
3.4 KiB
Python
92 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Dry-run first; remove only old, unreferenced objects or stale upload attempts."""
|
|
import argparse
|
|
from datetime import datetime, timedelta, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from db import ApiError, Store
|
|
from s3store import ObjectStore, S3Unavailable
|
|
|
|
|
|
def parse_timestamp(value):
|
|
return datetime.fromisoformat(value.replace('Z', '+00:00'))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--grace-seconds', type=int, default=900)
|
|
parser.add_argument('--apply', action='store_true')
|
|
args = parser.parse_args()
|
|
if args.grace_seconds < 900:
|
|
parser.error('--grace-seconds must be at least 900')
|
|
|
|
store = Store.connect(run_migrations=False)
|
|
objects = ObjectStore.from_env()
|
|
try:
|
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds)
|
|
records = store.list_attachments_for_reconciliation()
|
|
by_key = {record['object_key']: record for record in records}
|
|
summary = {
|
|
'mode': 'apply' if args.apply else 'dry-run',
|
|
'grace_seconds': args.grace_seconds,
|
|
'stale_uploads': 0,
|
|
'old_failed_objects': 0,
|
|
'orphan_objects': 0,
|
|
'missing_ready_objects': 0,
|
|
'actions_applied': 0,
|
|
}
|
|
|
|
for record in records:
|
|
key = record['object_key']
|
|
if (record['status'] == 'processing'
|
|
and parse_timestamp(record['created_at']) < cutoff):
|
|
summary['stale_uploads'] += 1
|
|
exists = objects.object_exists(key)
|
|
if args.apply:
|
|
if exists:
|
|
objects.delete_object(key)
|
|
store.mark_attachment_failed(
|
|
record['id'], 'upload_interrupted', datetime.now(timezone.utc)
|
|
)
|
|
summary['actions_applied'] += 1
|
|
elif (record['status'] == 'failed'
|
|
and parse_timestamp(record['updated_at']) < cutoff):
|
|
if objects.object_exists(key):
|
|
summary['old_failed_objects'] += 1
|
|
if args.apply:
|
|
objects.delete_object(key)
|
|
summary['actions_applied'] += 1
|
|
elif record['status'] == 'ready' and not objects.object_exists(key):
|
|
summary['missing_ready_objects'] += 1
|
|
if args.apply:
|
|
store.mark_reconciled_missing(
|
|
record['id'], datetime.now(timezone.utc)
|
|
)
|
|
summary['actions_applied'] += 1
|
|
|
|
for item in objects.list_objects():
|
|
key = item['Key']
|
|
if key in by_key or item['LastModified'] >= cutoff:
|
|
continue
|
|
summary['orphan_objects'] += 1
|
|
if args.apply:
|
|
objects.delete_object(key)
|
|
summary['actions_applied'] += 1
|
|
|
|
print(json.dumps(summary, sort_keys=True))
|
|
finally:
|
|
store.close()
|
|
objects.client.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (ApiError, S3Unavailable) as error:
|
|
print('Reconciliation failed: ' + error.message if isinstance(error, ApiError) else
|
|
'Reconciliation failed: S3 is unavailable', file=sys.stderr)
|
|
raise SystemExit(1) from None
|