Files
mtusi2026lab4/scripts/research_consistency.py

160 lines
8.3 KiB
Python

"""Track 2: repeatable faults, safe repair, protection and scan-cost measurements.
Run with automatic reconciler stopped, inside the isolated lab4 ops container.
"""
import hashlib
import json
from pathlib import Path
import statistics
import subprocess
import sys
import time
import uuid
from unittest.mock import patch
import psycopg
from botocore.exceptions import ClientError
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from app import store,object_store
from attachments import object_lock
from reconcile import reconcile
from client import Client,check
def main():
store.pool.open(wait=True,timeout=15)
s3=object_store.client
bucket=object_store.bucket
c=Client('http://lb:8080',timeout=20)
alice,bob,eve,chat=c.bootstrap()
data=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
sha=hashlib.sha256(data).hexdigest()
def key():
return 'originals/'+str(uuid.uuid4())
def put(k,body=data):
s3.put_object(Bucket=bucket,Key=k,Body=body,ContentType='image/png',Metadata={'sha256':hashlib.sha256(body).hexdigest()})
def age(aid):
with store.connect() as conn:
conn.execute("UPDATE attachments SET created_at=NOW()-interval '10 minutes',updated_at=NOW()-interval '10 minutes' WHERE id=%s",(aid,))
def states():
with store.connect() as conn:
return conn.execute('SELECT id,state,error_code FROM attachments ORDER BY id').fetchall()
def present(k):
try:
s3.head_object(Bucket=bucket,Key=k)
return True
except ClientError as exc:
if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'):
return False
raise
def sweep(apply=False,grace=5):
return reconcile(object_store,apply=apply,grace_seconds=grace,allow_short_grace=True)
missing,_=c.attachment(alice,chat)
c.message(alice,chat,'missing-object fixture',attachments=[missing['id']])
used,_=c.attachment(alice,chat)
c.message(alice,chat,'preserve-used fixture',attachments=[used['id']])
damaged,_=c.attachment(alice,chat)
c.message(alice,chat,'corrupt-object fixture',attachments=[damaged['id']])
# Simulate stale DB restore/loss of an unreferenced metadata row.
orphan,_=c.attachment(alice,chat)
orphan_key='originals/'+orphan['id']
with store.connect() as conn:
conn.execute('DELETE FROM attachments WHERE id=%s',(orphan['id'],))
crash=subprocess.run([sys.executable,'scripts/fault_after_put.py',alice['id'],chat['id']],capture_output=True,text=True)
check(crash.returncode==86,'fault process did not terminate at the real PUT/COMMIT boundary')
crashed=json.loads(crash.stdout)
check(present(crashed['key']),'PUT did not precede crash')
empty_id=str(uuid.uuid4())
# Another failure shape: intent was committed, but S3 PUT never succeeded.
with store.connect() as conn:
conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,created_at,updated_at) VALUES (%s,%s,%s,%s,'no-put.png','image/png',%s,%s,'pending',NOW()-interval '10 minutes',NOW()-interval '10 minutes')",
(empty_id,chat['id'],alice['id'],'originals/'+empty_id,len(data),sha))
for aid in (missing['id'],used['id'],damaged['id'],crashed['id']):
age(aid)
s3.delete_object(Bucket=bucket,Key='originals/'+missing['id'])
put('originals/'+damaged['id'],b'corrupt bytes')
locked_key=key();put(locked_key)
time.sleep(6) # Real S3 timestamps age; no fake server clock or lifecycle override.
fresh,_=c.attachment(alice,chat)
fresh_orphan=key();put(fresh_orphan)
before=states()
with store.connect() as uploader:
object_lock(uploader,locked_key)
dry=sweep()
check(states()==before,'dry-run modified metadata')
check(present(orphan_key) and present(crashed['key']),'dry-run deleted objects')
required={('delete_orphan',orphan_key),('abort_pending',crashed['key']),
('abort_pending','originals/'+empty_id),('missing_object','originals/'+missing['id']),
('corrupt_object','originals/'+damaged['id'])}
observed={(a['kind'],a['key']) for a in dry['actions']}
check(required<=observed,'dry-run missed an injected failure')
check(dry['skipped_busy']>=1,'active upload lock ignored')
put(fresh_orphan) # Freshness is measured separately at each run's cutoff.
applied=sweep(True)
check(present(locked_key),'in-flight locked key deleted')
check(present(fresh_orphan),'fresh object deleted')
check(present('originals/'+fresh['id']),'fresh ready attachment deleted')
check(present('originals/'+used['id']),'used ready attachment deleted')
check(not present(orphan_key) and not present(crashed['key']),'old orphan/pending object not cleaned')
for aid,expected in ((crashed['id'],'incomplete_upload'),(empty_id,'incomplete_upload'),(missing['id'],'missing_object'),(damaged['id'],'corrupt_object')):
with store.connect() as conn:
row=object_store.fetch(aid,conn)
check(row['state']=='failed' and row['error_code']==expected,'incorrect repair state')
c.call('GET','/api/v1/attachments/'+missing['id']+'/content',409,actor=bob)
with store.connect() as conn:
check(conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s',(missing['id'],)).fetchone() is not None,'missing metadata/usage link deleted')
# Known test-only locked orphan is removed after proving lock protection.
s3.delete_object(Bucket=bucket,Key=locked_key)
put(fresh_orphan)
repeated=sweep(True)
check(not repeated['actions'],'second apply is not idempotent')
put('originals/'+missing['id'])
age(missing['id'])
recovered=sweep(True)
check(any(a['kind']=='restore_ready' and a['key']=='originals/'+missing['id'] for a in recovered['actions']),'restored object not made available')
check(c.call('GET','/api/v1/attachments/'+missing['id']+'/content',actor=bob)['bytes']==data,'restoration changed original')
# Errors are not evidence of absence: both failures must stop cleanup.
before=states()
failures=[]
with patch.object(s3,'head_object',side_effect=ClientError({'Error':{'Code':'ServiceUnavailable'},'ResponseMetadata':{'HTTPStatusCode':503}},'HeadObject')):
try:
sweep(True)
raise AssertionError('S3 503 was swallowed')
except ClientError:
failures.append('s3_503_aborts_without_missing_inference')
with patch.object(store,'connect',side_effect=psycopg.OperationalError('injected database outage')):
try:
sweep(True)
raise AssertionError('DB error was swallowed')
except psycopg.OperationalError:
failures.append('db_outage_aborts_before_s3_mutation')
check(states()==before,'dependency error changed attachment states')
s3.delete_object(Bucket=bucket,Key=fresh_orphan)
protection={'fresh_orphan_preserved':True,'fresh_ready_preserved':True,'used_ready_preserved':True,
'active_lock_preserved':True,'missing_metadata_and_links_retained':True,
'dry_run_no_mutation':True,'second_apply_actions':len(repeated['actions'])}
# Cost: independent orphan fixtures added to the same baseline, dry-run only.
costs=[]
cost_keys=[]
time.sleep(6) # Age the fixed baseline before timing the zero-extra-object group.
try:
for target in (0,50,200):
for _ in range(target-len(cost_keys)):
k=key();put(k);cost_keys.append(k)
if target:
time.sleep(6)
for repeat in range(3):
row=sweep()
costs.append({'extra_orphans':target,'repeat':repeat+1,**row})
finally:
for k in cost_keys:
s3.delete_object(Bucket=bucket,Key=k)
print(json.dumps({'status':'PASS','grace_seconds_in_isolated_test':5,'production_grace_seconds':300,
'real_process_crash_exit':crash.returncode,'dry_run':dry,'applied':applied,
'repeated':repeated,'recovered':recovered,'protection':protection,
'dependency_failure_checks':failures,'scan_cost_runs':costs},indent=2))
store.pool.close();object_store.close()
if __name__=='__main__':
main()