forked from mtusi2026/mtusi2026lab4
121 lines
6.9 KiB
Python
121 lines
6.9 KiB
Python
"""Inject a real S3/DB split, missing object, dry-run/apply/repeat and age safety."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
import httpx
|
|
from stack import ROOT, docker, ready
|
|
|
|
OUT=ROOT/'evidence'
|
|
OUT.mkdir(exist_ok=True)
|
|
result={}
|
|
|
|
def sql(q): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',q)
|
|
def inside(code):
|
|
return subprocess.check_output(['docker','compose','exec','-T','api','python','-'],
|
|
input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip()
|
|
def reconcile(apply=False):
|
|
args=['compose','exec','-T','api','python','-m','src.reconcile','--grace-seconds','60']
|
|
if apply: args.append('--apply')
|
|
return json.loads(docker(*args))
|
|
def objects():
|
|
return set(json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps([o['Key'] for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix='originals/') for o in p.get('Contents',[])]))")))
|
|
|
|
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
encoded=base64.b64encode(data).decode()
|
|
digest=hashlib.sha256(data).hexdigest()
|
|
|
|
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
|
|
ready(c)
|
|
name='reconcile_'+uuid.uuid4().hex[:18]
|
|
password='reconciliation-test-123'
|
|
r=c.post('/api/v1/users',json={'username':name,'display_name':'Reconcile','password':password}); assert r.status_code==201,r.text
|
|
uid=r.json()['id']
|
|
r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201
|
|
h={'Authorization':'Bearer '+r.json()['token']}
|
|
r=c.post('/api/v1/chats',headers=h,json={'title':'Reconciliation','member_ids':[uid]}); assert r.status_code==201
|
|
chat=r.json()['id']
|
|
def upload(filename):
|
|
return c.post(f'/api/v1/chats/{chat}/attachments',headers={**h,'Content-Type':'image/png'},params={'filename':filename},content=data)
|
|
used=upload('used.png'); missing=upload('missing.png')
|
|
assert used.status_code==missing.status_code==201
|
|
used,missing=used.json(),missing.json()
|
|
r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[used['id']]})
|
|
assert r.status_code==201
|
|
used_key=sql("SELECT object_key FROM attachments WHERE id='"+used['id']+"'")
|
|
missing_key=sql("SELECT object_key FROM attachments WHERE id='"+missing['id']+"'")
|
|
filename='fault_'+uuid.uuid4().hex+'.png'
|
|
trigger='fault_'+uuid.uuid4().hex
|
|
before=objects()
|
|
# Scope is one unique synthetic filename; unrelated requests remain unaffected.
|
|
sql(f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='{filename}' THEN RAISE EXCEPTION 'synthetic transient failure' USING ERRCODE='40001'; END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {trigger}();")
|
|
try:
|
|
failed=upload(filename)
|
|
assert failed.status_code==503,(failed.status_code,failed.text)
|
|
finally:
|
|
sql(f'DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}();')
|
|
orphaned=objects()-before
|
|
assert len(orphaned)==1,orphaned
|
|
orphan=next(iter(orphaned)); orphan_created=time.monotonic()
|
|
assert sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"'")=='0'
|
|
result['put_before_database_failure']={'http_status':503,'orphan':orphan,'metadata_rows':0}
|
|
# Fresh orphan must remain even with --apply.
|
|
fresh_run=reconcile(True)
|
|
assert orphan in objects()
|
|
result['fresh_orphan_preserved']=fresh_run
|
|
# Generate 200 real object/metadata pairs for a measurable traversal; no mock S3.
|
|
setup_code=f'''import base64,io,uuid,os,psycopg
|
|
from src.storage import Storage
|
|
s=Storage(); data=base64.b64decode({encoded!r})
|
|
with psycopg.connect(os.environ['DATABASE_URL']) as db:
|
|
for n in range(200):
|
|
aid=uuid.uuid4(); key='originals/'+str(aid)
|
|
s.put(key,io.BytesIO(data),len(data),'image/png',{digest!r})
|
|
db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)',(aid,{chat!r},{uid!r},'scan-fixture.png',key,'image/png',len(data),{digest!r}))
|
|
print('Created 200 synthetic object/metadata pairs')'''
|
|
result['dataset']=inside(setup_code)
|
|
inside("from src.storage import Storage; Storage().delete("+repr(missing_key)+")")
|
|
before_objects=objects()
|
|
dry=reconcile(False)
|
|
assert objects()==before_objects
|
|
assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='ready'
|
|
assert any(a['action']=='quarantine_missing' and a['id']==missing['id'] for a in dry['actions'])
|
|
result['missing_dry_run']=dry
|
|
applied=reconcile(True)
|
|
assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='missing'
|
|
assert c.get('/api/v1/attachments/'+missing['id'],headers=h).status_code==503
|
|
assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data
|
|
result['missing_quarantined']=applied
|
|
# Restore from the known test fixture (in reality from backup), then verify byte checksum before publishing ready.
|
|
inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(missing_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")")
|
|
result['restored_from_fixture']=reconcile(True)
|
|
assert c.get('/api/v1/attachments/'+missing['id']+'/content',headers=h).content==data
|
|
# Real elapsed grace, not backdated timestamps or a monkeypatched clock.
|
|
while time.monotonic()-orphan_created<65:
|
|
time.sleep(min(5,65-(time.monotonic()-orphan_created)))
|
|
fresh_key='originals/'+str(uuid.uuid4())
|
|
inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(fresh_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")")
|
|
before_objects=objects()
|
|
dry=reconcile(False)
|
|
assert objects()==before_objects
|
|
assert any(a=={'action':'delete_orphan','key':orphan} for a in dry['actions'])
|
|
assert not any(a.get('key') in (fresh_key,used_key,missing_key) for a in dry['actions'])
|
|
result['aged_orphan_dry_run']=dry
|
|
result['aged_orphan_apply']=reconcile(True)
|
|
after=objects()
|
|
assert orphan not in after and fresh_key in after and used_key in after and missing_key in after
|
|
repeated=reconcile(True)
|
|
assert repeated['actions']==[],repeated
|
|
result['repeat_noop']=repeated
|
|
assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data
|
|
result['used_and_fresh_preserved']=True
|
|
result['actual_grace_elapsed_seconds']=round(time.monotonic()-orphan_created,3)
|
|
result['passed']=True
|
|
# Remove only the known test-only fresh orphan after the assertions; not application data.
|
|
inside("from src.storage import Storage; Storage().delete("+repr(fresh_key)+")")
|
|
(OUT/'reconciliation.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(result,indent=2))
|