forked from mtusi2026/mtusi2026lab6
180 lines
11 KiB
Python
180 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens."""
|
|
import argparse
|
|
import concurrent.futures
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
FAULTS=ROOT/'.lab6-faults'
|
|
COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')]
|
|
RESULTS=[]
|
|
SELECTED=set()
|
|
|
|
def compose(*args, allow_failure=False):
|
|
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180)
|
|
if p.returncode and not allow_failure:
|
|
# Infrastructure logs may contain settings; report only action and exit status.
|
|
raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode))
|
|
return p
|
|
|
|
def sql(query):
|
|
return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip()
|
|
|
|
def uid(value):
|
|
return str(uuid.UUID(value))
|
|
|
|
def job(att,version=1):
|
|
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version))
|
|
|
|
def operator(action,id=None,version=None, allow_failure=False):
|
|
args=['run','--rm','--no-deps','-T','operator','jobs',action]
|
|
if id: args+=['--id',uid(id)]
|
|
if version:args+=['--pipeline-version',str(version)]
|
|
return compose(*args,allow_failure=allow_failure)
|
|
|
|
def wait(predicate,timeout=65):
|
|
deadline=time.monotonic()+timeout
|
|
while time.monotonic()<deadline:
|
|
v=predicate()
|
|
if v:return v
|
|
time.sleep(.1)
|
|
raise Failure('experiment condition timed out')
|
|
|
|
def directive(point,id,action):
|
|
p=FAULTS/(point+'.'+id)
|
|
Path(str(p)+'.hit').unlink(missing_ok=True)
|
|
p.write_text(action);p.chmod(0o666)
|
|
return p
|
|
|
|
def hit(p,timeout=65):
|
|
wait(lambda:Path(str(p)+'.hit').exists(),timeout)
|
|
return Path(str(p)+'.hit').read_text().strip()
|
|
|
|
def metadata(c,actor,att):
|
|
return c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
|
|
|
|
def state(c,actor,att,value,timeout=65):
|
|
return wait(lambda:(a if (a:=metadata(c,actor,att))['status']==value else None),timeout)
|
|
|
|
def ready(c,actor,att,content):
|
|
a=c.wait_ready(actor,att)
|
|
check(a['error_code'] is None,'ready has error')
|
|
original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']
|
|
check(original==content,'original changed')
|
|
variants={v['name']:v for v in a['variants']}
|
|
check(set(variants)=={'thumbnail','crop','optimized'},'variant set differs')
|
|
for name,v in variants.items():
|
|
r=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
|
|
check(hashlib.sha256(r['bytes']).hexdigest()==v['sha256'] and len(r['bytes'])==v['size_bytes'],'variant checksum mismatch')
|
|
if name!='optimized':check(jpeg_size(r['bytes'])==(v['width'],v['height']),'JPEG dimensions differ')
|
|
else:check(r['bytes'][:4]==b'RIFF' and r['bytes'][8:12]==b'WEBP','optimized is not WebP')
|
|
rows=int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))
|
|
check(rows in (4,8),'manifest has duplicate/missing logical variants')
|
|
return a
|
|
|
|
def record(name,run,fn):
|
|
if SELECTED and name not in SELECTED:return
|
|
start=time.monotonic();detail=fn() or {}
|
|
r={'scenario':name,'run':run,'result':'PASS','seconds':round(time.monotonic()-start,3),**detail}
|
|
RESULTS.append(r);print(json.dumps(r),flush=True)
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);p.add_argument('--base-url',default='http://localhost:18086');p.add_argument('--output',default='evidence/pipeline.json');p.add_argument('--scenario',action='append',default=[]);args=p.parse_args()
|
|
SELECTED.update(args.scenario)
|
|
FAULTS.mkdir(exist_ok=True);FAULTS.chmod(0o777)
|
|
for f in FAULTS.iterdir():f.unlink()
|
|
compose('up','-d','--build','--wait','--wait-timeout','180')
|
|
c=Client(args.base_url,timeout=15);alice,bob,eve,chat=c.bootstrap()
|
|
def upload_stopped():
|
|
compose('stop','worker');att,content=c.attachment(alice,chat);return att,content,job(att)
|
|
try:
|
|
for run in range(1,args.repeats+1):
|
|
def recovery():
|
|
att,data,jid=upload_stopped();c.message(alice,chat,'',attachments=[att['id']]);c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
|
|
before=sql('SELECT pg_postmaster_start_time()')
|
|
compose('restart','api','postgres','redis','s3','broker')
|
|
wait(lambda:health(c),90);compose('run','--rm','--no-deps','-T','s3-init');compose('start','worker');ready(c,bob,att,data)
|
|
check(sql('SELECT pg_postmaster_start_time()')!=before,'PostgreSQL was not restarted')
|
|
return {'attachment_id':att['id'],'job_id':jid,'postgres_restarted':True}
|
|
record('worker_down_and_store_restart',run,recovery)
|
|
for point in ('before_objects','after_original_put','after_objects','after_result_commit','before_ack'):
|
|
def crash(point=point):
|
|
att,data,jid=upload_stopped();marker=directive(point,jid,'crash');compose('start','worker');hit(marker);ready(c,bob,att,data)
|
|
return {'job_id':jid,'manifest_rows':int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))}
|
|
record('crash_'+point,run,crash)
|
|
def duplicates():
|
|
att,data=c.attachment(alice,chat);first=ready(c,bob,att,data);jid=job(att)
|
|
for _ in range(3):operator('redeliver',jid)
|
|
time.sleep(.5);check(metadata(c,bob,att)==first,'ready changed after duplicate')
|
|
return {'job_id':jid,'copies':3}
|
|
record('ready_redelivery',run,duplicates)
|
|
def publish_crash():
|
|
att,data,jid=upload_stopped();compose('stop','dispatcher');marker=directive('after_publish',jid,'crash')
|
|
# Existing publish may have completed before stop. Add a second durable notification.
|
|
sql("UPDATE outbox SET status='pending',next_attempt_at=clock_timestamp() WHERE job_id='%s'"%uid(jid))
|
|
compose('start','dispatcher');hit(marker);compose('start','worker');ready(c,bob,att,data)
|
|
wait(lambda:sql("SELECT count(*) FROM outbox WHERE job_id='%s' AND status='published'"%uid(jid))!='0',45)
|
|
return {'job_id':jid}
|
|
record('crash_confirm_before_outbox_mark',run,publish_crash)
|
|
def version_fence():
|
|
att,data,jid=upload_stopped();marker=directive('after_objects',jid,'pause');compose('start','worker');hit(marker)
|
|
# Recovery expires the real 8s lease; a newer attempt finishes on the second worker slot.
|
|
first=ready(c,bob,att,data);check(int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid)))>=2,'lease not recovered')
|
|
operator('reprocess',att['id'],2);second=ready(c,bob,att,data)
|
|
check(first['variants']!=second['variants'],'pipeline versions are indistinguishable')
|
|
operator('redeliver',jid);marker.unlink();time.sleep(1)
|
|
check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2')
|
|
check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back')
|
|
ready(c,bob,att,data)
|
|
return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8}
|
|
record('lease_reorder_and_stale_version',run,version_fence)
|
|
def retries():
|
|
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65)
|
|
check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored')
|
|
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
|
marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed')
|
|
return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6}
|
|
record('bounded_retry_and_explicit_replay',run,retries)
|
|
def permanent():
|
|
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid}
|
|
record('permanent_error',run,permanent)
|
|
for point in ('after_staging','after_upload_commit'):
|
|
def api_crash(point=point):
|
|
if point=='after_upload_commit':compose('stop','dispatcher')
|
|
marker=directive(point,'all','crash')
|
|
try:c.attachment(alice,chat)
|
|
except Failure:pass
|
|
aid=hit(marker);att={'id':aid}
|
|
if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists')
|
|
else:
|
|
check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached')
|
|
compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes())
|
|
wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'}
|
|
record('api_crash_'+point,run,api_crash)
|
|
payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS}
|
|
check(bool(RESULTS),'no scenarios selected')
|
|
payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
|
|
(ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n')
|
|
finally:
|
|
for f in FAULTS.iterdir():f.unlink()
|
|
compose('stop','worker','dispatcher',allow_failure=True)
|
|
# Restore standard configuration, disabling fault injection and short experiment leases.
|
|
subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180)
|
|
|
|
def health(c):
|
|
try:c.call('GET','/health/ready',schema='Health');return True
|
|
except Failure:return False
|
|
|
|
if __name__=='__main__':main()
|