Files
mtusi2026lab6/scripts/experiment-slo.py
T
2026-11-17 13:08:13 +03:00

148 lines
10 KiB
Python

#!/usr/bin/env python3
"""Bounded SLO experiment on the isolated lab6 stand, with raw observations.
No synthetic metrics or time acceleration. Restores every injected fault in finally.
"""
import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client
from observations import get,query,scalar
C=[str(ROOT/'scripts/compose.sh')]
def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90)
def start_service(name):
ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split()
if not ids:raise RuntimeError('Missing existing '+name+' container')
subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None
class Traffic:
def __init__(self):
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
self.chat=chat;self.token=a['token'];self.actor=a;self.client=c
self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes()
self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64)
def request(self,index,scheduled,upload=False):
try:
path='/api/v1/chats/'+self.chat['id']
headers={'Authorization':'Bearer '+self.token};body=None
if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png'
elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex})
else:method='GET';path+='/messages?limit=50'
r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method)
status=0;att=None;message_id=None
try:
try:response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as e:response=e
with response:
status=response.status;data=response.read()
if upload and status==202:att=json.loads(data)['id']
if not upload and method=='POST' and status==201:message_id=json.loads(data)['id']
except (OSError,TimeoutError):pass
with self.lock:
self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000))
if att:self.attachments.append(att)
if message_id:self.messages.append(message_id)
finally:self.slots.release()
def run(self):
start=time.monotonic();i=0
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
while not self.stop.is_set():
scheduled=start+i*.1
if self.stop.wait(max(0,scheduled-time.monotonic())):break
if self.slots.acquire(False):pool.submit(self.request,i,scheduled)
else:
with self.lock:self.dropped+=1
if i%20==0:
if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True)
else:
with self.lock:self.dropped+=1
i+=1
def summary(self):
with self.lock:rows=list(self.rows);dropped=self.dropped
xs=[r[3] for r in rows if 200<=r[2]<300]
return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}}
def main():
def interrupted(*_):raise KeyboardInterrupt()
signal.signal(signal.SIGTERM,interrupted)
p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args()
out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]}
out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
target=ROOT/a.output
def save():target.write_text(json.dumps(out,indent=2)+'\n')
traffic=Traffic();t=threading.Thread(target=traffic.run);t.start()
def phase(name,seconds):
at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save()
before=traffic.summary();nextstats=0;nextprint=0
while time.time()<at+seconds:
now=time.time()
row={'at':now,'phase':name}
try:
row.update({'alerts':get('prometheus','/api/v1/alerts')['data']['alerts'],'pending':scalar('sum(messenger_jobs_pending)'), 'db_lock_waiting':scalar('sum(messenger_db_locks_waiting)'), 'queue_age':scalar('max(messenger_queue_oldest_age_seconds)'), 'api_burn':scalar('messenger:api_error_ratio:1m / 0.01'), 'ready_burn':scalar('messenger:ready_error_ratio:1m / 0.01')})
except Exception as e:row['collection_error']=type(e).__name__
out['observations'].append(row)
if now>=nextstats:
ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split()
if ids:
raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20)
out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]})
nextstats=now+30
if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60
save();time.sleep(min(5,max(0,at+seconds-time.time())))
with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()]
lat=[r[3] for r in rows if 200<=r[2]<300]
elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows)
out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save()
def worker(duration,name):
compose('stop','worker')
try:phase(name,duration)
finally:start_service('worker')
def database(duration,name):
# A separate bounded transaction blocks only the local message table.
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
deadline=time.time()+10
while time.time()<deadline:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_slo_injection' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
time.sleep(.1)
else:raise RuntimeError('DB injection did not acquire lock')
phase(name,duration)
finally:
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'")
proc.wait(timeout=10)
try:
phase('baseline',a.warmup)
for i in range(1,a.repeats+1):
worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60)
database(5,f'db_short_{i}');phase(f'db_short_recovery_{i}',60)
worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown)
database(a.long,f'db_long_{i}');phase(f'db_recovery_{i}',a.cooldown)
if a.backend:
out['backend_before']={n:query(n) for n in ['otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_queue_size','otelcol_receiver_accepted_spans_total']}
compose('stop','tempo')
try:phase('tempo_unavailable',a.backend)
finally:start_service('tempo')
phase('tempo_recovery',60)
out['backend_after']={n:query(n) for n in out['backend_before']}
traffic.stop.set();t.join(timeout=15)
phase('final_drain',30)
# Every accepted upload must have a published result after recovery.
with traffic.lock:ids=list(traffic.attachments)
out['accepted_ids_count']=len(ids)
counts={}
for id in ids:
state=traffic.client.call('GET','/api/v1/attachments/'+id,actor=traffic.actor,schema='Attachment')['json']['status'];counts[state]=counts.get(state,0)+1
out['attachment_states']=counts
stored=set(sql("SELECT id FROM messages WHERE chat_id='%s'"%str(uuid.UUID(traffic.chat['id']))).splitlines())
with traffic.lock:confirmed=set(traffic.messages)
out['message_persistence']={'confirmed':len(confirmed),'stored':len(stored),'confirmed_missing':len(confirmed-stored)}
if confirmed-stored or counts.get('ready',0)!=len(ids):raise RuntimeError('Confirmed business data not preserved')
out['completed']=True
except BaseException as e:
out['completed']=False;out['failure_type']=type(e).__name__;raise
finally:
traffic.stop.set();t.join(timeout=15)
out['finished']=time.time();out['traffic']=traffic.summary();save()
if __name__=='__main__':main()