forked from mtusi2026/mtusi2026lab4
160 lines
8.3 KiB
Python
160 lines
8.3 KiB
Python
"""Track 2: repeatable faults, safe repair, protection and scan-cost measurements.
|
|
|
|
Run with automatic reconciler stopped, inside the isolated lab4 ops container.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from unittest.mock import patch
|
|
import psycopg
|
|
from botocore.exceptions import ClientError
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
|
from app import store,object_store
|
|
from attachments import object_lock
|
|
from reconcile import reconcile
|
|
from client import Client,check
|
|
|
|
|
|
def main():
|
|
store.pool.open(wait=True,timeout=15)
|
|
s3=object_store.client
|
|
bucket=object_store.bucket
|
|
c=Client('http://lb:8080',timeout=20)
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
data=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
|
|
sha=hashlib.sha256(data).hexdigest()
|
|
def key():
|
|
return 'originals/'+str(uuid.uuid4())
|
|
def put(k,body=data):
|
|
s3.put_object(Bucket=bucket,Key=k,Body=body,ContentType='image/png',Metadata={'sha256':hashlib.sha256(body).hexdigest()})
|
|
def age(aid):
|
|
with store.connect() as conn:
|
|
conn.execute("UPDATE attachments SET created_at=NOW()-interval '10 minutes',updated_at=NOW()-interval '10 minutes' WHERE id=%s",(aid,))
|
|
def states():
|
|
with store.connect() as conn:
|
|
return conn.execute('SELECT id,state,error_code FROM attachments ORDER BY id').fetchall()
|
|
def present(k):
|
|
try:
|
|
s3.head_object(Bucket=bucket,Key=k)
|
|
return True
|
|
except ClientError as exc:
|
|
if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'):
|
|
return False
|
|
raise
|
|
def sweep(apply=False,grace=5):
|
|
return reconcile(object_store,apply=apply,grace_seconds=grace,allow_short_grace=True)
|
|
missing,_=c.attachment(alice,chat)
|
|
c.message(alice,chat,'missing-object fixture',attachments=[missing['id']])
|
|
used,_=c.attachment(alice,chat)
|
|
c.message(alice,chat,'preserve-used fixture',attachments=[used['id']])
|
|
damaged,_=c.attachment(alice,chat)
|
|
c.message(alice,chat,'corrupt-object fixture',attachments=[damaged['id']])
|
|
# Simulate stale DB restore/loss of an unreferenced metadata row.
|
|
orphan,_=c.attachment(alice,chat)
|
|
orphan_key='originals/'+orphan['id']
|
|
with store.connect() as conn:
|
|
conn.execute('DELETE FROM attachments WHERE id=%s',(orphan['id'],))
|
|
crash=subprocess.run([sys.executable,'scripts/fault_after_put.py',alice['id'],chat['id']],capture_output=True,text=True)
|
|
check(crash.returncode==86,'fault process did not terminate at the real PUT/COMMIT boundary')
|
|
crashed=json.loads(crash.stdout)
|
|
check(present(crashed['key']),'PUT did not precede crash')
|
|
empty_id=str(uuid.uuid4())
|
|
# Another failure shape: intent was committed, but S3 PUT never succeeded.
|
|
with store.connect() as conn:
|
|
conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,created_at,updated_at) VALUES (%s,%s,%s,%s,'no-put.png','image/png',%s,%s,'pending',NOW()-interval '10 minutes',NOW()-interval '10 minutes')",
|
|
(empty_id,chat['id'],alice['id'],'originals/'+empty_id,len(data),sha))
|
|
for aid in (missing['id'],used['id'],damaged['id'],crashed['id']):
|
|
age(aid)
|
|
s3.delete_object(Bucket=bucket,Key='originals/'+missing['id'])
|
|
put('originals/'+damaged['id'],b'corrupt bytes')
|
|
locked_key=key();put(locked_key)
|
|
time.sleep(6) # Real S3 timestamps age; no fake server clock or lifecycle override.
|
|
fresh,_=c.attachment(alice,chat)
|
|
fresh_orphan=key();put(fresh_orphan)
|
|
before=states()
|
|
with store.connect() as uploader:
|
|
object_lock(uploader,locked_key)
|
|
dry=sweep()
|
|
check(states()==before,'dry-run modified metadata')
|
|
check(present(orphan_key) and present(crashed['key']),'dry-run deleted objects')
|
|
required={('delete_orphan',orphan_key),('abort_pending',crashed['key']),
|
|
('abort_pending','originals/'+empty_id),('missing_object','originals/'+missing['id']),
|
|
('corrupt_object','originals/'+damaged['id'])}
|
|
observed={(a['kind'],a['key']) for a in dry['actions']}
|
|
check(required<=observed,'dry-run missed an injected failure')
|
|
check(dry['skipped_busy']>=1,'active upload lock ignored')
|
|
put(fresh_orphan) # Freshness is measured separately at each run's cutoff.
|
|
applied=sweep(True)
|
|
check(present(locked_key),'in-flight locked key deleted')
|
|
check(present(fresh_orphan),'fresh object deleted')
|
|
check(present('originals/'+fresh['id']),'fresh ready attachment deleted')
|
|
check(present('originals/'+used['id']),'used ready attachment deleted')
|
|
check(not present(orphan_key) and not present(crashed['key']),'old orphan/pending object not cleaned')
|
|
for aid,expected in ((crashed['id'],'incomplete_upload'),(empty_id,'incomplete_upload'),(missing['id'],'missing_object'),(damaged['id'],'corrupt_object')):
|
|
with store.connect() as conn:
|
|
row=object_store.fetch(aid,conn)
|
|
check(row['state']=='failed' and row['error_code']==expected,'incorrect repair state')
|
|
c.call('GET','/api/v1/attachments/'+missing['id']+'/content',409,actor=bob)
|
|
with store.connect() as conn:
|
|
check(conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s',(missing['id'],)).fetchone() is not None,'missing metadata/usage link deleted')
|
|
# Known test-only locked orphan is removed after proving lock protection.
|
|
s3.delete_object(Bucket=bucket,Key=locked_key)
|
|
put(fresh_orphan)
|
|
repeated=sweep(True)
|
|
check(not repeated['actions'],'second apply is not idempotent')
|
|
put('originals/'+missing['id'])
|
|
age(missing['id'])
|
|
recovered=sweep(True)
|
|
check(any(a['kind']=='restore_ready' and a['key']=='originals/'+missing['id'] for a in recovered['actions']),'restored object not made available')
|
|
check(c.call('GET','/api/v1/attachments/'+missing['id']+'/content',actor=bob)['bytes']==data,'restoration changed original')
|
|
# Errors are not evidence of absence: both failures must stop cleanup.
|
|
before=states()
|
|
failures=[]
|
|
with patch.object(s3,'head_object',side_effect=ClientError({'Error':{'Code':'ServiceUnavailable'},'ResponseMetadata':{'HTTPStatusCode':503}},'HeadObject')):
|
|
try:
|
|
sweep(True)
|
|
raise AssertionError('S3 503 was swallowed')
|
|
except ClientError:
|
|
failures.append('s3_503_aborts_without_missing_inference')
|
|
with patch.object(store,'connect',side_effect=psycopg.OperationalError('injected database outage')):
|
|
try:
|
|
sweep(True)
|
|
raise AssertionError('DB error was swallowed')
|
|
except psycopg.OperationalError:
|
|
failures.append('db_outage_aborts_before_s3_mutation')
|
|
check(states()==before,'dependency error changed attachment states')
|
|
s3.delete_object(Bucket=bucket,Key=fresh_orphan)
|
|
protection={'fresh_orphan_preserved':True,'fresh_ready_preserved':True,'used_ready_preserved':True,
|
|
'active_lock_preserved':True,'missing_metadata_and_links_retained':True,
|
|
'dry_run_no_mutation':True,'second_apply_actions':len(repeated['actions'])}
|
|
# Cost: independent orphan fixtures added to the same baseline, dry-run only.
|
|
costs=[]
|
|
cost_keys=[]
|
|
time.sleep(6) # Age the fixed baseline before timing the zero-extra-object group.
|
|
try:
|
|
for target in (0,50,200):
|
|
for _ in range(target-len(cost_keys)):
|
|
k=key();put(k);cost_keys.append(k)
|
|
if target:
|
|
time.sleep(6)
|
|
for repeat in range(3):
|
|
row=sweep()
|
|
costs.append({'extra_orphans':target,'repeat':repeat+1,**row})
|
|
finally:
|
|
for k in cost_keys:
|
|
s3.delete_object(Bucket=bucket,Key=k)
|
|
print(json.dumps({'status':'PASS','grace_seconds_in_isolated_test':5,'production_grace_seconds':300,
|
|
'real_process_crash_exit':crash.returncode,'dry_run':dry,'applied':applied,
|
|
'repeated':repeated,'recovered':recovered,'protection':protection,
|
|
'dependency_failure_checks':failures,'scan_cost_runs':costs},indent=2))
|
|
store.pool.close();object_store.close()
|
|
|
|
if __name__=='__main__':
|
|
main()
|