Files
mtusi2026lab4/scripts/reconciliation_experiment.py
T

121 lines
6.9 KiB
Python

"""Inject a real S3/DB split, missing object, dry-run/apply/repeat and age safety."""
import base64
import hashlib
import json
import subprocess
import time
import uuid
import httpx
from stack import ROOT, docker, ready
OUT=ROOT/'evidence'
OUT.mkdir(exist_ok=True)
result={}
def sql(q): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',q)
def inside(code):
return subprocess.check_output(['docker','compose','exec','-T','api','python','-'],
input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip()
def reconcile(apply=False):
args=['compose','exec','-T','api','python','-m','src.reconcile','--grace-seconds','60']
if apply: args.append('--apply')
return json.loads(docker(*args))
def objects():
return set(json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps([o['Key'] for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix='originals/') for o in p.get('Contents',[])]))")))
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
encoded=base64.b64encode(data).decode()
digest=hashlib.sha256(data).hexdigest()
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
ready(c)
name='reconcile_'+uuid.uuid4().hex[:18]
password='reconciliation-test-123'
r=c.post('/api/v1/users',json={'username':name,'display_name':'Reconcile','password':password}); assert r.status_code==201,r.text
uid=r.json()['id']
r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201
h={'Authorization':'Bearer '+r.json()['token']}
r=c.post('/api/v1/chats',headers=h,json={'title':'Reconciliation','member_ids':[uid]}); assert r.status_code==201
chat=r.json()['id']
def upload(filename):
return c.post(f'/api/v1/chats/{chat}/attachments',headers={**h,'Content-Type':'image/png'},params={'filename':filename},content=data)
used=upload('used.png'); missing=upload('missing.png')
assert used.status_code==missing.status_code==201
used,missing=used.json(),missing.json()
r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[used['id']]})
assert r.status_code==201
used_key=sql("SELECT object_key FROM attachments WHERE id='"+used['id']+"'")
missing_key=sql("SELECT object_key FROM attachments WHERE id='"+missing['id']+"'")
filename='fault_'+uuid.uuid4().hex+'.png'
trigger='fault_'+uuid.uuid4().hex
before=objects()
# Scope is one unique synthetic filename; unrelated requests remain unaffected.
sql(f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='{filename}' THEN RAISE EXCEPTION 'synthetic transient failure' USING ERRCODE='40001'; END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {trigger}();")
try:
failed=upload(filename)
assert failed.status_code==503,(failed.status_code,failed.text)
finally:
sql(f'DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}();')
orphaned=objects()-before
assert len(orphaned)==1,orphaned
orphan=next(iter(orphaned)); orphan_created=time.monotonic()
assert sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"'")=='0'
result['put_before_database_failure']={'http_status':503,'orphan':orphan,'metadata_rows':0}
# Fresh orphan must remain even with --apply.
fresh_run=reconcile(True)
assert orphan in objects()
result['fresh_orphan_preserved']=fresh_run
# Generate 200 real object/metadata pairs for a measurable traversal; no mock S3.
setup_code=f'''import base64,io,uuid,os,psycopg
from src.storage import Storage
s=Storage(); data=base64.b64decode({encoded!r})
with psycopg.connect(os.environ['DATABASE_URL']) as db:
for n in range(200):
aid=uuid.uuid4(); key='originals/'+str(aid)
s.put(key,io.BytesIO(data),len(data),'image/png',{digest!r})
db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)',(aid,{chat!r},{uid!r},'scan-fixture.png',key,'image/png',len(data),{digest!r}))
print('Created 200 synthetic object/metadata pairs')'''
result['dataset']=inside(setup_code)
inside("from src.storage import Storage; Storage().delete("+repr(missing_key)+")")
before_objects=objects()
dry=reconcile(False)
assert objects()==before_objects
assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='ready'
assert any(a['action']=='quarantine_missing' and a['id']==missing['id'] for a in dry['actions'])
result['missing_dry_run']=dry
applied=reconcile(True)
assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='missing'
assert c.get('/api/v1/attachments/'+missing['id'],headers=h).status_code==503
assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data
result['missing_quarantined']=applied
# Restore from the known test fixture (in reality from backup), then verify byte checksum before publishing ready.
inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(missing_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")")
result['restored_from_fixture']=reconcile(True)
assert c.get('/api/v1/attachments/'+missing['id']+'/content',headers=h).content==data
# Real elapsed grace, not backdated timestamps or a monkeypatched clock.
while time.monotonic()-orphan_created<65:
time.sleep(min(5,65-(time.monotonic()-orphan_created)))
fresh_key='originals/'+str(uuid.uuid4())
inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(fresh_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")")
before_objects=objects()
dry=reconcile(False)
assert objects()==before_objects
assert any(a=={'action':'delete_orphan','key':orphan} for a in dry['actions'])
assert not any(a.get('key') in (fresh_key,used_key,missing_key) for a in dry['actions'])
result['aged_orphan_dry_run']=dry
result['aged_orphan_apply']=reconcile(True)
after=objects()
assert orphan not in after and fresh_key in after and used_key in after and missing_key in after
repeated=reconcile(True)
assert repeated['actions']==[],repeated
result['repeat_noop']=repeated
assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data
result['used_and_fresh_preserved']=True
result['actual_grace_elapsed_seconds']=round(time.monotonic()-orphan_created,3)
result['passed']=True
# Remove only the known test-only fresh orphan after the assertions; not application data.
inside("from src.storage import Storage; Storage().delete("+repr(fresh_key)+")")
(OUT/'reconciliation.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8')
print(json.dumps(result,indent=2))