"""Track 2: repeatable faults, safe repair, protection and scan-cost measurements. Run with automatic reconciler stopped, inside the isolated lab4 ops container. """ import hashlib import json from pathlib import Path import statistics import subprocess import sys import time import uuid from unittest.mock import patch import psycopg from botocore.exceptions import ClientError sys.path.insert(0,str(Path(__file__).resolve().parents[1])) sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) from app import store,object_store from attachments import object_lock from reconcile import reconcile from client import Client,check def main(): store.pool.open(wait=True,timeout=15) s3=object_store.client bucket=object_store.bucket c=Client('http://lb:8080',timeout=20) alice,bob,eve,chat=c.bootstrap() data=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() sha=hashlib.sha256(data).hexdigest() def key(): return 'originals/'+str(uuid.uuid4()) def put(k,body=data): s3.put_object(Bucket=bucket,Key=k,Body=body,ContentType='image/png',Metadata={'sha256':hashlib.sha256(body).hexdigest()}) def age(aid): with store.connect() as conn: conn.execute("UPDATE attachments SET created_at=NOW()-interval '10 minutes',updated_at=NOW()-interval '10 minutes' WHERE id=%s",(aid,)) def states(): with store.connect() as conn: return conn.execute('SELECT id,state,error_code FROM attachments ORDER BY id').fetchall() def present(k): try: s3.head_object(Bucket=bucket,Key=k) return True except ClientError as exc: if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'): return False raise def sweep(apply=False,grace=5): return reconcile(object_store,apply=apply,grace_seconds=grace,allow_short_grace=True) missing,_=c.attachment(alice,chat) c.message(alice,chat,'missing-object fixture',attachments=[missing['id']]) used,_=c.attachment(alice,chat) c.message(alice,chat,'preserve-used fixture',attachments=[used['id']]) damaged,_=c.attachment(alice,chat) c.message(alice,chat,'corrupt-object fixture',attachments=[damaged['id']]) # Simulate stale DB restore/loss of an unreferenced metadata row. orphan,_=c.attachment(alice,chat) orphan_key='originals/'+orphan['id'] with store.connect() as conn: conn.execute('DELETE FROM attachments WHERE id=%s',(orphan['id'],)) crash=subprocess.run([sys.executable,'scripts/fault_after_put.py',alice['id'],chat['id']],capture_output=True,text=True) check(crash.returncode==86,'fault process did not terminate at the real PUT/COMMIT boundary') crashed=json.loads(crash.stdout) check(present(crashed['key']),'PUT did not precede crash') empty_id=str(uuid.uuid4()) # Another failure shape: intent was committed, but S3 PUT never succeeded. with store.connect() as conn: conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,created_at,updated_at) VALUES (%s,%s,%s,%s,'no-put.png','image/png',%s,%s,'pending',NOW()-interval '10 minutes',NOW()-interval '10 minutes')", (empty_id,chat['id'],alice['id'],'originals/'+empty_id,len(data),sha)) for aid in (missing['id'],used['id'],damaged['id'],crashed['id']): age(aid) s3.delete_object(Bucket=bucket,Key='originals/'+missing['id']) put('originals/'+damaged['id'],b'corrupt bytes') locked_key=key();put(locked_key) time.sleep(6) # Real S3 timestamps age; no fake server clock or lifecycle override. fresh,_=c.attachment(alice,chat) fresh_orphan=key();put(fresh_orphan) before=states() with store.connect() as uploader: object_lock(uploader,locked_key) dry=sweep() check(states()==before,'dry-run modified metadata') check(present(orphan_key) and present(crashed['key']),'dry-run deleted objects') required={('delete_orphan',orphan_key),('abort_pending',crashed['key']), ('abort_pending','originals/'+empty_id),('missing_object','originals/'+missing['id']), ('corrupt_object','originals/'+damaged['id'])} observed={(a['kind'],a['key']) for a in dry['actions']} check(required<=observed,'dry-run missed an injected failure') check(dry['skipped_busy']>=1,'active upload lock ignored') put(fresh_orphan) # Freshness is measured separately at each run's cutoff. applied=sweep(True) check(present(locked_key),'in-flight locked key deleted') check(present(fresh_orphan),'fresh object deleted') check(present('originals/'+fresh['id']),'fresh ready attachment deleted') check(present('originals/'+used['id']),'used ready attachment deleted') check(not present(orphan_key) and not present(crashed['key']),'old orphan/pending object not cleaned') for aid,expected in ((crashed['id'],'incomplete_upload'),(empty_id,'incomplete_upload'),(missing['id'],'missing_object'),(damaged['id'],'corrupt_object')): with store.connect() as conn: row=object_store.fetch(aid,conn) check(row['state']=='failed' and row['error_code']==expected,'incorrect repair state') c.call('GET','/api/v1/attachments/'+missing['id']+'/content',409,actor=bob) with store.connect() as conn: check(conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s',(missing['id'],)).fetchone() is not None,'missing metadata/usage link deleted') # Known test-only locked orphan is removed after proving lock protection. s3.delete_object(Bucket=bucket,Key=locked_key) put(fresh_orphan) repeated=sweep(True) check(not repeated['actions'],'second apply is not idempotent') put('originals/'+missing['id']) age(missing['id']) recovered=sweep(True) check(any(a['kind']=='restore_ready' and a['key']=='originals/'+missing['id'] for a in recovered['actions']),'restored object not made available') check(c.call('GET','/api/v1/attachments/'+missing['id']+'/content',actor=bob)['bytes']==data,'restoration changed original') # Errors are not evidence of absence: both failures must stop cleanup. before=states() failures=[] with patch.object(s3,'head_object',side_effect=ClientError({'Error':{'Code':'ServiceUnavailable'},'ResponseMetadata':{'HTTPStatusCode':503}},'HeadObject')): try: sweep(True) raise AssertionError('S3 503 was swallowed') except ClientError: failures.append('s3_503_aborts_without_missing_inference') with patch.object(store,'connect',side_effect=psycopg.OperationalError('injected database outage')): try: sweep(True) raise AssertionError('DB error was swallowed') except psycopg.OperationalError: failures.append('db_outage_aborts_before_s3_mutation') check(states()==before,'dependency error changed attachment states') s3.delete_object(Bucket=bucket,Key=fresh_orphan) protection={'fresh_orphan_preserved':True,'fresh_ready_preserved':True,'used_ready_preserved':True, 'active_lock_preserved':True,'missing_metadata_and_links_retained':True, 'dry_run_no_mutation':True,'second_apply_actions':len(repeated['actions'])} # Cost: independent orphan fixtures added to the same baseline, dry-run only. costs=[] cost_keys=[] time.sleep(6) # Age the fixed baseline before timing the zero-extra-object group. try: for target in (0,50,200): for _ in range(target-len(cost_keys)): k=key();put(k);cost_keys.append(k) if target: time.sleep(6) for repeat in range(3): row=sweep() costs.append({'extra_orphans':target,'repeat':repeat+1,**row}) finally: for k in cost_keys: s3.delete_object(Bucket=bucket,Key=k) print(json.dumps({'status':'PASS','grace_seconds_in_isolated_test':5,'production_grace_seconds':300, 'real_process_crash_exit':crash.returncode,'dry_run':dry,'applied':applied, 'repeated':repeated,'recovered':recovered,'protection':protection, 'dependency_failure_checks':failures,'scan_cost_runs':costs},indent=2)) store.pool.close();object_store.close() if __name__=='__main__': main()