Files
mtusi2026lab6/student-tests/pipeline.py
T
2026-11-17 13:08:13 +03:00

180 lines
11 KiB
Python

#!/usr/bin/env python3
"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens."""
import argparse
import concurrent.futures
import datetime
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
FAULTS=ROOT/'.lab6-faults'
COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')]
RESULTS=[]
SELECTED=set()
def compose(*args, allow_failure=False):
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180)
if p.returncode and not allow_failure:
# Infrastructure logs may contain settings; report only action and exit status.
raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode))
return p
def sql(query):
return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip()
def uid(value):
return str(uuid.UUID(value))
def job(att,version=1):
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version))
def operator(action,id=None,version=None, allow_failure=False):
args=['run','--rm','--no-deps','-T','operator','jobs',action]
if id: args+=['--id',uid(id)]
if version:args+=['--pipeline-version',str(version)]
return compose(*args,allow_failure=allow_failure)
def wait(predicate,timeout=65):
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
v=predicate()
if v:return v
time.sleep(.1)
raise Failure('experiment condition timed out')
def directive(point,id,action):
p=FAULTS/(point+'.'+id)
Path(str(p)+'.hit').unlink(missing_ok=True)
p.write_text(action);p.chmod(0o666)
return p
def hit(p,timeout=65):
wait(lambda:Path(str(p)+'.hit').exists(),timeout)
return Path(str(p)+'.hit').read_text().strip()
def metadata(c,actor,att):
return c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
def state(c,actor,att,value,timeout=65):
return wait(lambda:(a if (a:=metadata(c,actor,att))['status']==value else None),timeout)
def ready(c,actor,att,content):
a=c.wait_ready(actor,att)
check(a['error_code'] is None,'ready has error')
original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']
check(original==content,'original changed')
variants={v['name']:v for v in a['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'variant set differs')
for name,v in variants.items():
r=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
check(hashlib.sha256(r['bytes']).hexdigest()==v['sha256'] and len(r['bytes'])==v['size_bytes'],'variant checksum mismatch')
if name!='optimized':check(jpeg_size(r['bytes'])==(v['width'],v['height']),'JPEG dimensions differ')
else:check(r['bytes'][:4]==b'RIFF' and r['bytes'][8:12]==b'WEBP','optimized is not WebP')
rows=int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))
check(rows in (4,8),'manifest has duplicate/missing logical variants')
return a
def record(name,run,fn):
if SELECTED and name not in SELECTED:return
start=time.monotonic();detail=fn() or {}
r={'scenario':name,'run':run,'result':'PASS','seconds':round(time.monotonic()-start,3),**detail}
RESULTS.append(r);print(json.dumps(r),flush=True)
def main():
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);p.add_argument('--base-url',default='http://localhost:18086');p.add_argument('--output',default='evidence/pipeline.json');p.add_argument('--scenario',action='append',default=[]);args=p.parse_args()
SELECTED.update(args.scenario)
FAULTS.mkdir(exist_ok=True);FAULTS.chmod(0o777)
for f in FAULTS.iterdir():f.unlink()
compose('up','-d','--build','--wait','--wait-timeout','180')
c=Client(args.base_url,timeout=15);alice,bob,eve,chat=c.bootstrap()
def upload_stopped():
compose('stop','worker');att,content=c.attachment(alice,chat);return att,content,job(att)
try:
for run in range(1,args.repeats+1):
def recovery():
att,data,jid=upload_stopped();c.message(alice,chat,'',attachments=[att['id']]);c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
before=sql('SELECT pg_postmaster_start_time()')
compose('restart','api','postgres','redis','s3','broker')
wait(lambda:health(c),90);compose('run','--rm','--no-deps','-T','s3-init');compose('start','worker');ready(c,bob,att,data)
check(sql('SELECT pg_postmaster_start_time()')!=before,'PostgreSQL was not restarted')
return {'attachment_id':att['id'],'job_id':jid,'postgres_restarted':True}
record('worker_down_and_store_restart',run,recovery)
for point in ('before_objects','after_original_put','after_objects','after_result_commit','before_ack'):
def crash(point=point):
att,data,jid=upload_stopped();marker=directive(point,jid,'crash');compose('start','worker');hit(marker);ready(c,bob,att,data)
return {'job_id':jid,'manifest_rows':int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'"%uid(att['id'])))}
record('crash_'+point,run,crash)
def duplicates():
att,data=c.attachment(alice,chat);first=ready(c,bob,att,data);jid=job(att)
for _ in range(3):operator('redeliver',jid)
time.sleep(.5);check(metadata(c,bob,att)==first,'ready changed after duplicate')
return {'job_id':jid,'copies':3}
record('ready_redelivery',run,duplicates)
def publish_crash():
att,data,jid=upload_stopped();compose('stop','dispatcher');marker=directive('after_publish',jid,'crash')
# Existing publish may have completed before stop. Add a second durable notification.
sql("UPDATE outbox SET status='pending',next_attempt_at=clock_timestamp() WHERE job_id='%s'"%uid(jid))
compose('start','dispatcher');hit(marker);compose('start','worker');ready(c,bob,att,data)
wait(lambda:sql("SELECT count(*) FROM outbox WHERE job_id='%s' AND status='published'"%uid(jid))!='0',45)
return {'job_id':jid}
record('crash_confirm_before_outbox_mark',run,publish_crash)
def version_fence():
att,data,jid=upload_stopped();marker=directive('after_objects',jid,'pause');compose('start','worker');hit(marker)
# Recovery expires the real 8s lease; a newer attempt finishes on the second worker slot.
first=ready(c,bob,att,data);check(int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid)))>=2,'lease not recovered')
operator('reprocess',att['id'],2);second=ready(c,bob,att,data)
check(first['variants']!=second['variants'],'pipeline versions are indistinguishable')
operator('redeliver',jid);marker.unlink();time.sleep(1)
check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2')
check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back')
ready(c,bob,att,data)
return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8}
record('lease_reorder_and_stale_version',run,version_fence)
def retries():
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65)
check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed')
return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6}
record('bounded_retry_and_explicit_replay',run,retries)
def permanent():
att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid}
record('permanent_error',run,permanent)
for point in ('after_staging','after_upload_commit'):
def api_crash(point=point):
if point=='after_upload_commit':compose('stop','dispatcher')
marker=directive(point,'all','crash')
try:c.attachment(alice,chat)
except Failure:pass
aid=hit(marker);att={'id':aid}
if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists')
else:
check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached')
compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes())
wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'}
record('api_crash_'+point,run,api_crash)
payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS}
check(bool(RESULTS),'no scenarios selected')
payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
(ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n')
finally:
for f in FAULTS.iterdir():f.unlink()
compose('stop','worker','dispatcher',allow_failure=True)
# Restore standard configuration, disabling fault injection and short experiment leases.
subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180)
def health(c):
try:c.call('GET','/health/ready',schema='Health');return True
except Failure:return False
if __name__=='__main__':main()