"""Inject a real S3/DB split, missing object, dry-run/apply/repeat and age safety.""" import base64 import hashlib import json import subprocess import time import uuid import httpx from stack import ROOT, docker, ready OUT=ROOT/'evidence' OUT.mkdir(exist_ok=True) result={} def sql(q): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',q) def inside(code): return subprocess.check_output(['docker','compose','exec','-T','api','python','-'], input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip() def reconcile(apply=False): args=['compose','exec','-T','api','python','-m','src.reconcile','--grace-seconds','60'] if apply: args.append('--apply') return json.loads(docker(*args)) def objects(): return set(json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps([o['Key'] for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix='originals/') for o in p.get('Contents',[])]))"))) data=(ROOT/'tests/fixtures/sample.png').read_bytes() encoded=base64.b64encode(data).decode() digest=hashlib.sha256(data).hexdigest() with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c: ready(c) name='reconcile_'+uuid.uuid4().hex[:18] password='reconciliation-test-123' r=c.post('/api/v1/users',json={'username':name,'display_name':'Reconcile','password':password}); assert r.status_code==201,r.text uid=r.json()['id'] r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201 h={'Authorization':'Bearer '+r.json()['token']} r=c.post('/api/v1/chats',headers=h,json={'title':'Reconciliation','member_ids':[uid]}); assert r.status_code==201 chat=r.json()['id'] def upload(filename): return c.post(f'/api/v1/chats/{chat}/attachments',headers={**h,'Content-Type':'image/png'},params={'filename':filename},content=data) used=upload('used.png'); missing=upload('missing.png') assert used.status_code==missing.status_code==201 used,missing=used.json(),missing.json() r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[used['id']]}) assert r.status_code==201 used_key=sql("SELECT object_key FROM attachments WHERE id='"+used['id']+"'") missing_key=sql("SELECT object_key FROM attachments WHERE id='"+missing['id']+"'") filename='fault_'+uuid.uuid4().hex+'.png' trigger='fault_'+uuid.uuid4().hex before=objects() # Scope is one unique synthetic filename; unrelated requests remain unaffected. sql(f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='{filename}' THEN RAISE EXCEPTION 'synthetic transient failure' USING ERRCODE='40001'; END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {trigger}();") try: failed=upload(filename) assert failed.status_code==503,(failed.status_code,failed.text) finally: sql(f'DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}();') orphaned=objects()-before assert len(orphaned)==1,orphaned orphan=next(iter(orphaned)); orphan_created=time.monotonic() assert sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"'")=='0' result['put_before_database_failure']={'http_status':503,'orphan':orphan,'metadata_rows':0} # Fresh orphan must remain even with --apply. fresh_run=reconcile(True) assert orphan in objects() result['fresh_orphan_preserved']=fresh_run # Generate 200 real object/metadata pairs for a measurable traversal; no mock S3. setup_code=f'''import base64,io,uuid,os,psycopg from src.storage import Storage s=Storage(); data=base64.b64decode({encoded!r}) with psycopg.connect(os.environ['DATABASE_URL']) as db: for n in range(200): aid=uuid.uuid4(); key='originals/'+str(aid) s.put(key,io.BytesIO(data),len(data),'image/png',{digest!r}) db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)',(aid,{chat!r},{uid!r},'scan-fixture.png',key,'image/png',len(data),{digest!r})) print('Created 200 synthetic object/metadata pairs')''' result['dataset']=inside(setup_code) inside("from src.storage import Storage; Storage().delete("+repr(missing_key)+")") before_objects=objects() dry=reconcile(False) assert objects()==before_objects assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='ready' assert any(a['action']=='quarantine_missing' and a['id']==missing['id'] for a in dry['actions']) result['missing_dry_run']=dry applied=reconcile(True) assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='missing' assert c.get('/api/v1/attachments/'+missing['id'],headers=h).status_code==503 assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data result['missing_quarantined']=applied # Restore from the known test fixture (in reality from backup), then verify byte checksum before publishing ready. inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(missing_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")") result['restored_from_fixture']=reconcile(True) assert c.get('/api/v1/attachments/'+missing['id']+'/content',headers=h).content==data # Real elapsed grace, not backdated timestamps or a monkeypatched clock. while time.monotonic()-orphan_created<65: time.sleep(min(5,65-(time.monotonic()-orphan_created))) fresh_key='originals/'+str(uuid.uuid4()) inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(fresh_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")") before_objects=objects() dry=reconcile(False) assert objects()==before_objects assert any(a=={'action':'delete_orphan','key':orphan} for a in dry['actions']) assert not any(a.get('key') in (fresh_key,used_key,missing_key) for a in dry['actions']) result['aged_orphan_dry_run']=dry result['aged_orphan_apply']=reconcile(True) after=objects() assert orphan not in after and fresh_key in after and used_key in after and missing_key in after repeated=reconcile(True) assert repeated['actions']==[],repeated result['repeat_noop']=repeated assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data result['used_and_fresh_preserved']=True result['actual_grace_elapsed_seconds']=round(time.monotonic()-orphan_created,3) result['passed']=True # Remove only the known test-only fresh orphan after the assertions; not application data. inside("from src.storage import Storage; Storage().delete("+repr(fresh_key)+")") (OUT/'reconciliation.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8') print(json.dumps(result,indent=2))