Files

163 lines
8.1 KiB
Python

"""Repeatable DB/S3 reconciler. Dry-run by default; no ready object is deleted."""
import argparse
from datetime import datetime, timedelta, timezone
import json
import time
import uuid
from botocore.exceptions import ClientError
from psycopg.rows import dict_row
from attachments import object_lock
def managed_key(key):
try:
return key == 'originals/' + str(uuid.UUID(key.removeprefix('originals/')))
except ValueError:
return False
def reconcile(objects, *, apply=False, grace_seconds=300, allow_short_grace=False, prefix='originals/'):
if grace_seconds < 1:
raise ValueError("grace must be positive")
if grace_seconds < 60 and not allow_short_grace:
raise ValueError('production grace must be >=60s; tests must explicitly allow a shorter grace')
if prefix != 'originals/':
raise ValueError('only the managed originals/ namespace may be scanned')
store, s3 = objects.store, objects.client
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
started = time.monotonic()
result = {'mode': 'apply' if apply else 'dry-run', 'grace_seconds': grace_seconds,
'objects_scanned': 0, 'metadata_scanned': 0, 's3_list_calls': 0,
's3_head_calls': 0, 's3_delete_calls': 0, 'skipped_fresh': 0,
'skipped_busy': 0, 'skipped_in_use': 0, 'action_counts': {}, 'actions_truncated': 0, 'actions': []}
def head(key):
result['s3_head_calls'] += 1
try:
return s3.head_object(Bucket=objects.bucket, Key=key)
except ClientError as exc:
if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'):
return None
raise # 403/5xx/timeouts are not proof of a missing object.
def action(kind, key, conn, row=None):
result['action_counts'][kind] = result['action_counts'].get(kind, 0) + 1
if len(result['actions']) < 1000:
result['actions'].append({'kind': kind, 'key': key})
else:
result['actions_truncated'] += 1
if not apply:
return
if kind in ('delete_orphan','abort_pending'):
if head(key) is not None:
s3.delete_object(Bucket=objects.bucket, Key=key)
result['s3_delete_calls'] += 1
if row is not None:
conn.execute("UPDATE attachments SET state='failed',error_code='incomplete_upload',updated_at=NOW() WHERE id=%s", (row['id'],))
elif kind in ('missing_object','corrupt_object'):
conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=NOW() WHERE id=%s", (kind,row['id']))
elif kind == 'restore_ready':
conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (row['id'],))
# One run at a time; object-level transaction locks are shared with uploads.
with store.connect() as guard:
acquired = guard.execute('SELECT pg_try_advisory_lock(202604)').fetchone()[0]
if not acquired:
raise RuntimeError('another reconciler is running')
try:
paginator = s3.get_paginator('list_objects_v2')
for page in paginator.paginate(Bucket=objects.bucket, Prefix=prefix, PaginationConfig={'PageSize':200}):
result['s3_list_calls'] += 1
for item in page.get('Contents', []):
result['objects_scanned'] += 1
key = item['Key']
if not managed_key(key):
continue
if item['LastModified'] > cutoff:
result['skipped_fresh'] += 1
continue
with store.connect() as conn:
if not object_lock(conn,key,wait=False):
result['skipped_busy'] += 1
continue
# Re-read BOTH stores under lock instead of trusting an old list snapshot.
aid = key.split('/')[1]
row = objects.fetch(aid,conn)
current = head(key)
if current is None:
continue
if current['LastModified'] > cutoff:
result['skipped_fresh'] += 1
continue
if conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',(aid,)).fetchone():
result['skipped_in_use'] += 1
continue
if row is None:
action('delete_orphan',key,conn)
elif row['state']=='pending' and row['updated_at'] <= cutoff:
action('abort_pending',key,conn,row)
# Includes pending rows whose S3 PUT never occurred; keyset pages keep
# scan RAM bounded. Failed missing/corrupt rows can recover if restored.
last = '00000000-0000-0000-0000-000000000000'
while True:
with store.connect() as conn, conn.cursor(row_factory=dict_row) as cur:
cur.execute("SELECT id FROM attachments WHERE id>%s AND (state IN ('ready','pending') OR error_code IN ('missing_object','corrupt_object')) ORDER BY id LIMIT 200", (last,))
rows = cur.fetchall()
if not rows:
break
for item in rows:
last = str(item['id'])
result['metadata_scanned'] += 1
with store.connect() as conn:
key = 'originals/' + last
if not object_lock(conn,key,wait=False):
result['skipped_busy'] += 1
continue
row = objects.fetch(last,conn)
if row is None:
continue
if row['updated_at'] > cutoff:
result['skipped_fresh'] += 1
continue
current = head(key)
if row['state']=='pending':
if conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',(last,)).fetchone():
result['skipped_in_use'] += 1
continue
if current is not None and current['LastModified'] > cutoff:
result['skipped_fresh'] += 1
continue
action('abort_pending',key,conn,row)
elif current is None:
if row['state']!='failed' or row['error_code']!='missing_object':
action('missing_object',key,conn,row)
elif current['ContentLength'] != row['size_bytes'] or current.get('Metadata',{}).get('sha256') != row['sha256']:
if row['state']!='failed' or row['error_code']!='corrupt_object':
action('corrupt_object',key,conn,row)
elif row['state']=='failed':
action('restore_ready',key,conn,row)
finally:
guard.execute('SELECT pg_advisory_unlock(202604)')
result['elapsed_seconds'] = round(time.monotonic()-started,6)
return result
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--apply',action='store_true')
parser.add_argument('--grace-seconds',type=int,default=300)
parser.add_argument('--allow-short-grace',action='store_true',help='isolated synthetic tests only')
args=parser.parse_args()
from app import store, object_store
store.pool.open(wait=True,timeout=15)
try:
print(json.dumps(reconcile(object_store,apply=args.apply,grace_seconds=args.grace_seconds,
allow_short_grace=args.allow_short_grace),indent=2))
finally:
store.pool.close()
object_store.close()
if __name__=='__main__':
main()