Files
mtusi2026lab6/student-tests/pipeline.py
T

82 lines
6.6 KiB
Python
Executable File

#!/usr/bin/env python3
"""Black-box fault experiment. Mutates only this synthetic Compose project."""
import hashlib,json,os,subprocess,sys,time,uuid
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check,Failure
BASE=os.environ.get('BASE_URL','http://localhost:8086');C=Client(BASE)
RESULT={'method':'real process/container faults; no forged timestamps','cases':[]}
def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip()
def sql(query):return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query)
def record(name,**kwargs):RESULT['cases'].append(dict(name=name,**kwargs));print('PASS:',name,flush=True)
def meta(a,id):return C.call('GET','/api/v1/attachments/'+id,actor=a,schema='Attachment')['json']
def wait_status(a,id,status,timeout=130):
start=time.monotonic()
while time.monotonic()-start<timeout:
try:item=meta(a,id)
except Failure:time.sleep(.25);continue
if item['status']==status:return item,round(time.monotonic()-start,3)
time.sleep(.25)
raise AssertionError('timeout waiting '+status+' last='+item['status'])
def job(id,version=1):return json.loads(sql("SELECT row_to_json(x) FROM (SELECT 1 schema_version,job_id,'attachment.process.v1' type,attachment_id,pipeline_version,1 attempt,to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"') created_at FROM pipeline_jobs WHERE attachment_id='"+id+"' AND pipeline_version="+str(version)+") x"))
def publish(payload):
# The URL is read inside the container; credentials never enter output/argv.
script='valkey-cli -u "$(cat /run/secrets/broker_url)" --no-auth-warning XADD attachment.jobs.v1 "*" payload "$1" >/dev/null'
return dc('exec','-T','broker','sh','-c',script,'sh',json.dumps(payload))
def content(a,id,variant='original'):return C.call('GET','/api/v1/attachments/'+id+'/content?variant='+variant,actor=a)['bytes']
def upload(a,chat):return C.attachment(a,chat)
def start_worker():dc('up','-d','--no-deps','worker')
def fault(a,chat,point):
dc('stop','worker');att,data=upload(a,chat);name='lab-fault-'+uuid.uuid4().hex[:10]
dc('run','-d','--no-deps','--name',name,'-e','LAB_FAULT_POINT='+point,'worker')
try:
deadline=time.monotonic()+45
while time.monotonic()<deadline:
if subprocess.run(['docker','exec',name,'test','-f','/tmp/fault-reached'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode==0:break
time.sleep(.2)
else:raise AssertionError('fault boundary never reached '+point)
status=meta(a,att['id'])['status'];check(status==('ready' if point=='after_commit' else 'processing'),'wrong crash boundary')
subprocess.run(['docker','kill',name],check=True,stdout=subprocess.DEVNULL)
start_worker();ready,recovery=wait_status(a,att['id'],'ready');check(content(a,att['id'])==data,'original changed after crash')
time.sleep(2)
check(int(sql("SELECT count(*) FROM pipeline_jobs WHERE attachment_id='"+att['id']+"'"))==1,'extra logical job')
record('crash_'+point,attachment_id=att['id'],status_at_crash=status,recovery_seconds=recovery,versions=len(ready['variants']))
return att,data
finally:subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL);start_worker()
def main():
a,b,e,chat=C.bootstrap()
dc('stop','worker','dispatcher');att,data=upload(a,chat)
C.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=a)
C.message(a,chat,'',attachments=[att['id']]);C.message(a,chat,'text while worker is down')
check(sql("SELECT published_at IS NULL FROM pipeline_jobs WHERE attachment_id='"+att['id']+"'")=='t','expected committed unpublished outbox')
dc('restart','api','broker');dc('up','-d','--no-deps','dispatcher','worker');ready,recovery=wait_status(a,att['id'],'ready')
check(content(a,att['id'])==data,'durable stage lost');check({v['name'] for v in ready['variants']}=={'thumbnail','crop','optimized'},'missing variants')
for v in ready['variants']:check(hashlib.sha256(content(a,att['id'],v['name'])).hexdigest()==v['sha256'],'variant checksum')
C.call('GET','/api/v1/attachments/'+att['id'],404,actor=e)
record('commit_before_publish_api_broker_restart',attachment_id=att['id'],recovery_seconds=recovery,original_identical=True)
# No ACK occurred for these real crashes. Lease + stream reclaim recover them.
for point in ['before_objects','after_objects','after_commit']:fault(a,chat,point)
before=meta(a,att['id']);p=job(att['id'])
for _ in range(5):publish(p)
time.sleep(2);check(meta(a,att['id'])==before,'ready duplicate changed metadata');record('five_duplicate_deliveries',unchanged=True)
dc('run','--rm','--no-deps','worker','reprocess',att['id']);v2,_=wait_status(a,att['id'],'ready');check(sql("SELECT pipeline_version FROM attachments WHERE id='"+att['id']+"'")=='2','version not incremented')
v2bytes={v['name']:content(a,att['id'],v['name']) for v in v2['variants']}
# Old version redelivery is reordered behind a fully committed new version.
for _ in range(3):publish(p)
time.sleep(2);check(meta(a,att['id'])==v2,'old version changed metadata');check(all(content(a,att['id'],k)==v for k,v in v2bytes.items()),'old version overwrote bytes')
record('reordered_v1_after_v2',metadata_unchanged=True,checksums_unchanged=True)
# A permanent checksum mismatch is observable and cannot be attached. Restore
# the expected digest and explicitly replay; the source itself stays durable.
dc('stop','worker');bad,data=upload(a,chat);original=bad['sha256'];sql("UPDATE attachments SET sha256=repeat('0',64) WHERE id='"+bad['id']+"'");start_worker();failed,_=wait_status(a,bad['id'],'failed')
check(failed['error_code']=='invalid_image','permanent error code')
C.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=a,body={'text':'','attachment_ids':[bad['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
sql("UPDATE attachments SET sha256='"+original+"' WHERE id='"+bad['id']+"'");dc('run','--rm','--no-deps','worker','replay',bad['id']);replayed,_=wait_status(a,bad['id'],'ready')
check(content(a,bad['id'])==data,'replay lost original');check(int(sql("SELECT count(*) FROM pipeline_audit WHERE job_id=(SELECT job_id FROM pipeline_jobs WHERE attachment_id='"+bad['id']+"') AND event='operator_replay'"))==1,'missing replay audit')
record('failed_registry_explicit_audited_replay',error_code=failed['error_code'],ready=True)
print('PASS: versioned at-least-once pipeline invariants; NOT exactly-once delivery')
if __name__=='__main__':
try:main()
finally:
dc('up','-d','--no-deps','worker','dispatcher')
(ROOT/'evidence/pipeline.json').write_text(json.dumps(RESULT,indent=2)+'\n')