#!/usr/bin/env python3 """Bounded SLO experiment on the isolated lab6 stand, with raw observations. No synthetic metrics or time acceleration. Restores every injected fault in finally. """ import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid from pathlib import Path ROOT=Path(__file__).resolve().parents[1] sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')] from client import Client from observations import get,query,scalar C=[str(ROOT/'scripts/compose.sh')] def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90) def start_service(name): ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split() if not ids:raise RuntimeError('Missing existing '+name+' container') subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30) def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip() def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None class Traffic: def __init__(self): c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap() self.chat=chat;self.token=a['token'];self.actor=a;self.client=c self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes() self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64) def request(self,index,scheduled,upload=False): try: path='/api/v1/chats/'+self.chat['id'] headers={'Authorization':'Bearer '+self.token};body=None if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png' elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}) else:method='GET';path+='/messages?limit=50' r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method) status=0;att=None;message_id=None try: try:response=urllib.request.urlopen(r,timeout=6) except urllib.error.HTTPError as e:response=e with response: status=response.status;data=response.read() if upload and status==202:att=json.loads(data)['id'] if not upload and method=='POST' and status==201:message_id=json.loads(data)['id'] except (OSError,TimeoutError):pass with self.lock: self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000)) if att:self.attachments.append(att) if message_id:self.messages.append(message_id) finally:self.slots.release() def run(self): start=time.monotonic();i=0 with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool: while not self.stop.is_set(): scheduled=start+i*.1 if self.stop.wait(max(0,scheduled-time.monotonic())):break if self.slots.acquire(False):pool.submit(self.request,i,scheduled) else: with self.lock:self.dropped+=1 if i%20==0: if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True) else: with self.lock:self.dropped+=1 i+=1 def summary(self): with self.lock:rows=list(self.rows);dropped=self.dropped xs=[r[3] for r in rows if 200<=r[2]<300] return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}} def main(): def interrupted(*_):raise KeyboardInterrupt() signal.signal(signal.SIGTERM,interrupted) p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args() out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]} out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip() target=ROOT/a.output def save():target.write_text(json.dumps(out,indent=2)+'\n') traffic=Traffic();t=threading.Thread(target=traffic.run);t.start() def phase(name,seconds): at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save() before=traffic.summary();nextstats=0;nextprint=0 while time.time()=nextstats: ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split() if ids: raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20) out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]}) nextstats=now+30 if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60 save();time.sleep(min(5,max(0,at+seconds-time.time()))) with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()] lat=[r[3] for r in rows if 200<=r[2]<300] elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows) out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save() def worker(duration,name): compose('stop','worker') try:phase(name,duration) finally:start_service('worker') def database(duration,name): # A separate bounded transaction blocks only the local message table. proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) try: deadline=time.time()+10 while time.time()