forked from mtusi2026/mtusi2026lab4
163 lines
8.1 KiB
Python
163 lines
8.1 KiB
Python
"""Repeatable DB/S3 reconciler. Dry-run by default; no ready object is deleted."""
|
|
import argparse
|
|
from datetime import datetime, timedelta, timezone
|
|
import json
|
|
import time
|
|
import uuid
|
|
|
|
from botocore.exceptions import ClientError
|
|
from psycopg.rows import dict_row
|
|
from attachments import object_lock
|
|
|
|
|
|
def managed_key(key):
|
|
try:
|
|
return key == 'originals/' + str(uuid.UUID(key.removeprefix('originals/')))
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def reconcile(objects, *, apply=False, grace_seconds=300, allow_short_grace=False, prefix='originals/'):
|
|
if grace_seconds < 1:
|
|
raise ValueError("grace must be positive")
|
|
if grace_seconds < 60 and not allow_short_grace:
|
|
raise ValueError('production grace must be >=60s; tests must explicitly allow a shorter grace')
|
|
if prefix != 'originals/':
|
|
raise ValueError('only the managed originals/ namespace may be scanned')
|
|
store, s3 = objects.store, objects.client
|
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
|
started = time.monotonic()
|
|
result = {'mode': 'apply' if apply else 'dry-run', 'grace_seconds': grace_seconds,
|
|
'objects_scanned': 0, 'metadata_scanned': 0, 's3_list_calls': 0,
|
|
's3_head_calls': 0, 's3_delete_calls': 0, 'skipped_fresh': 0,
|
|
'skipped_busy': 0, 'skipped_in_use': 0, 'action_counts': {}, 'actions_truncated': 0, 'actions': []}
|
|
|
|
def head(key):
|
|
result['s3_head_calls'] += 1
|
|
try:
|
|
return s3.head_object(Bucket=objects.bucket, Key=key)
|
|
except ClientError as exc:
|
|
if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'):
|
|
return None
|
|
raise # 403/5xx/timeouts are not proof of a missing object.
|
|
|
|
def action(kind, key, conn, row=None):
|
|
result['action_counts'][kind] = result['action_counts'].get(kind, 0) + 1
|
|
if len(result['actions']) < 1000:
|
|
result['actions'].append({'kind': kind, 'key': key})
|
|
else:
|
|
result['actions_truncated'] += 1
|
|
if not apply:
|
|
return
|
|
if kind in ('delete_orphan','abort_pending'):
|
|
if head(key) is not None:
|
|
s3.delete_object(Bucket=objects.bucket, Key=key)
|
|
result['s3_delete_calls'] += 1
|
|
if row is not None:
|
|
conn.execute("UPDATE attachments SET state='failed',error_code='incomplete_upload',updated_at=NOW() WHERE id=%s", (row['id'],))
|
|
elif kind in ('missing_object','corrupt_object'):
|
|
conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=NOW() WHERE id=%s", (kind,row['id']))
|
|
elif kind == 'restore_ready':
|
|
conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (row['id'],))
|
|
|
|
# One run at a time; object-level transaction locks are shared with uploads.
|
|
with store.connect() as guard:
|
|
acquired = guard.execute('SELECT pg_try_advisory_lock(202604)').fetchone()[0]
|
|
if not acquired:
|
|
raise RuntimeError('another reconciler is running')
|
|
try:
|
|
paginator = s3.get_paginator('list_objects_v2')
|
|
for page in paginator.paginate(Bucket=objects.bucket, Prefix=prefix, PaginationConfig={'PageSize':200}):
|
|
result['s3_list_calls'] += 1
|
|
for item in page.get('Contents', []):
|
|
result['objects_scanned'] += 1
|
|
key = item['Key']
|
|
if not managed_key(key):
|
|
continue
|
|
if item['LastModified'] > cutoff:
|
|
result['skipped_fresh'] += 1
|
|
continue
|
|
with store.connect() as conn:
|
|
if not object_lock(conn,key,wait=False):
|
|
result['skipped_busy'] += 1
|
|
continue
|
|
# Re-read BOTH stores under lock instead of trusting an old list snapshot.
|
|
aid = key.split('/')[1]
|
|
row = objects.fetch(aid,conn)
|
|
current = head(key)
|
|
if current is None:
|
|
continue
|
|
if current['LastModified'] > cutoff:
|
|
result['skipped_fresh'] += 1
|
|
continue
|
|
if conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',(aid,)).fetchone():
|
|
result['skipped_in_use'] += 1
|
|
continue
|
|
if row is None:
|
|
action('delete_orphan',key,conn)
|
|
elif row['state']=='pending' and row['updated_at'] <= cutoff:
|
|
action('abort_pending',key,conn,row)
|
|
# Includes pending rows whose S3 PUT never occurred; keyset pages keep
|
|
# scan RAM bounded. Failed missing/corrupt rows can recover if restored.
|
|
last = '00000000-0000-0000-0000-000000000000'
|
|
while True:
|
|
with store.connect() as conn, conn.cursor(row_factory=dict_row) as cur:
|
|
cur.execute("SELECT id FROM attachments WHERE id>%s AND (state IN ('ready','pending') OR error_code IN ('missing_object','corrupt_object')) ORDER BY id LIMIT 200", (last,))
|
|
rows = cur.fetchall()
|
|
if not rows:
|
|
break
|
|
for item in rows:
|
|
last = str(item['id'])
|
|
result['metadata_scanned'] += 1
|
|
with store.connect() as conn:
|
|
key = 'originals/' + last
|
|
if not object_lock(conn,key,wait=False):
|
|
result['skipped_busy'] += 1
|
|
continue
|
|
row = objects.fetch(last,conn)
|
|
if row is None:
|
|
continue
|
|
if row['updated_at'] > cutoff:
|
|
result['skipped_fresh'] += 1
|
|
continue
|
|
current = head(key)
|
|
if row['state']=='pending':
|
|
if conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',(last,)).fetchone():
|
|
result['skipped_in_use'] += 1
|
|
continue
|
|
if current is not None and current['LastModified'] > cutoff:
|
|
result['skipped_fresh'] += 1
|
|
continue
|
|
action('abort_pending',key,conn,row)
|
|
elif current is None:
|
|
if row['state']!='failed' or row['error_code']!='missing_object':
|
|
action('missing_object',key,conn,row)
|
|
elif current['ContentLength'] != row['size_bytes'] or current.get('Metadata',{}).get('sha256') != row['sha256']:
|
|
if row['state']!='failed' or row['error_code']!='corrupt_object':
|
|
action('corrupt_object',key,conn,row)
|
|
elif row['state']=='failed':
|
|
action('restore_ready',key,conn,row)
|
|
finally:
|
|
guard.execute('SELECT pg_advisory_unlock(202604)')
|
|
result['elapsed_seconds'] = round(time.monotonic()-started,6)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--apply',action='store_true')
|
|
parser.add_argument('--grace-seconds',type=int,default=300)
|
|
parser.add_argument('--allow-short-grace',action='store_true',help='isolated synthetic tests only')
|
|
args=parser.parse_args()
|
|
from app import store, object_store
|
|
store.pool.open(wait=True,timeout=15)
|
|
try:
|
|
print(json.dumps(reconcile(object_store,apply=args.apply,grace_seconds=args.grace_seconds,
|
|
allow_short_grace=args.allow_short_grace),indent=2))
|
|
finally:
|
|
store.pool.close()
|
|
object_store.close()
|
|
|
|
if __name__=='__main__':
|
|
main()
|