forked from mtusi2026/mtusi2026lab6
148 lines
10 KiB
Python
148 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded SLO experiment on the isolated lab6 stand, with raw observations.
|
|
No synthetic metrics or time acceleration. Restores every injected fault in finally.
|
|
"""
|
|
import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
|
from client import Client
|
|
from observations import get,query,scalar
|
|
C=[str(ROOT/'scripts/compose.sh')]
|
|
def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90)
|
|
def start_service(name):
|
|
ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split()
|
|
if not ids:raise RuntimeError('Missing existing '+name+' container')
|
|
subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
|
|
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
|
|
def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None
|
|
class Traffic:
|
|
def __init__(self):
|
|
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
|
self.chat=chat;self.token=a['token'];self.actor=a;self.client=c
|
|
self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64)
|
|
def request(self,index,scheduled,upload=False):
|
|
try:
|
|
path='/api/v1/chats/'+self.chat['id']
|
|
headers={'Authorization':'Bearer '+self.token};body=None
|
|
if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png'
|
|
elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex})
|
|
else:method='GET';path+='/messages?limit=50'
|
|
r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method)
|
|
status=0;att=None;message_id=None
|
|
try:
|
|
try:response=urllib.request.urlopen(r,timeout=6)
|
|
except urllib.error.HTTPError as e:response=e
|
|
with response:
|
|
status=response.status;data=response.read()
|
|
if upload and status==202:att=json.loads(data)['id']
|
|
if not upload and method=='POST' and status==201:message_id=json.loads(data)['id']
|
|
except (OSError,TimeoutError):pass
|
|
with self.lock:
|
|
self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000))
|
|
if att:self.attachments.append(att)
|
|
if message_id:self.messages.append(message_id)
|
|
finally:self.slots.release()
|
|
def run(self):
|
|
start=time.monotonic();i=0
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
|
|
while not self.stop.is_set():
|
|
scheduled=start+i*.1
|
|
if self.stop.wait(max(0,scheduled-time.monotonic())):break
|
|
if self.slots.acquire(False):pool.submit(self.request,i,scheduled)
|
|
else:
|
|
with self.lock:self.dropped+=1
|
|
if i%20==0:
|
|
if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True)
|
|
else:
|
|
with self.lock:self.dropped+=1
|
|
i+=1
|
|
def summary(self):
|
|
with self.lock:rows=list(self.rows);dropped=self.dropped
|
|
xs=[r[3] for r in rows if 200<=r[2]<300]
|
|
return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}}
|
|
|
|
def main():
|
|
def interrupted(*_):raise KeyboardInterrupt()
|
|
signal.signal(signal.SIGTERM,interrupted)
|
|
p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args()
|
|
out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]}
|
|
out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
|
|
target=ROOT/a.output
|
|
def save():target.write_text(json.dumps(out,indent=2)+'\n')
|
|
traffic=Traffic();t=threading.Thread(target=traffic.run);t.start()
|
|
def phase(name,seconds):
|
|
at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save()
|
|
before=traffic.summary();nextstats=0;nextprint=0
|
|
while time.time()<at+seconds:
|
|
now=time.time()
|
|
row={'at':now,'phase':name}
|
|
try:
|
|
row.update({'alerts':get('prometheus','/api/v1/alerts')['data']['alerts'],'pending':scalar('sum(messenger_jobs_pending)'), 'db_lock_waiting':scalar('sum(messenger_db_locks_waiting)'), 'queue_age':scalar('max(messenger_queue_oldest_age_seconds)'), 'api_burn':scalar('messenger:api_error_ratio:1m / 0.01'), 'ready_burn':scalar('messenger:ready_error_ratio:1m / 0.01')})
|
|
except Exception as e:row['collection_error']=type(e).__name__
|
|
out['observations'].append(row)
|
|
if now>=nextstats:
|
|
ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split()
|
|
if ids:
|
|
raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20)
|
|
out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]})
|
|
nextstats=now+30
|
|
if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60
|
|
save();time.sleep(min(5,max(0,at+seconds-time.time())))
|
|
with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()]
|
|
lat=[r[3] for r in rows if 200<=r[2]<300]
|
|
elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows)
|
|
out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save()
|
|
def worker(duration,name):
|
|
compose('stop','worker')
|
|
try:phase(name,duration)
|
|
finally:start_service('worker')
|
|
def database(duration,name):
|
|
# A separate bounded transaction blocks only the local message table.
|
|
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
try:
|
|
deadline=time.time()+10
|
|
while time.time()<deadline:
|
|
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_slo_injection' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
|
|
time.sleep(.1)
|
|
else:raise RuntimeError('DB injection did not acquire lock')
|
|
phase(name,duration)
|
|
finally:
|
|
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'")
|
|
proc.wait(timeout=10)
|
|
try:
|
|
phase('baseline',a.warmup)
|
|
for i in range(1,a.repeats+1):
|
|
worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60)
|
|
database(5,f'db_short_{i}');phase(f'db_short_recovery_{i}',60)
|
|
worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown)
|
|
database(a.long,f'db_long_{i}');phase(f'db_recovery_{i}',a.cooldown)
|
|
if a.backend:
|
|
out['backend_before']={n:query(n) for n in ['otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_queue_size','otelcol_receiver_accepted_spans_total']}
|
|
compose('stop','tempo')
|
|
try:phase('tempo_unavailable',a.backend)
|
|
finally:start_service('tempo')
|
|
phase('tempo_recovery',60)
|
|
out['backend_after']={n:query(n) for n in out['backend_before']}
|
|
traffic.stop.set();t.join(timeout=15)
|
|
phase('final_drain',30)
|
|
# Every accepted upload must have a published result after recovery.
|
|
with traffic.lock:ids=list(traffic.attachments)
|
|
out['accepted_ids_count']=len(ids)
|
|
counts={}
|
|
for id in ids:
|
|
state=traffic.client.call('GET','/api/v1/attachments/'+id,actor=traffic.actor,schema='Attachment')['json']['status'];counts[state]=counts.get(state,0)+1
|
|
out['attachment_states']=counts
|
|
stored=set(sql("SELECT id FROM messages WHERE chat_id='%s'"%str(uuid.UUID(traffic.chat['id']))).splitlines())
|
|
with traffic.lock:confirmed=set(traffic.messages)
|
|
out['message_persistence']={'confirmed':len(confirmed),'stored':len(stored),'confirmed_missing':len(confirmed-stored)}
|
|
if confirmed-stored or counts.get('ready',0)!=len(ids):raise RuntimeError('Confirmed business data not preserved')
|
|
out['completed']=True
|
|
except BaseException as e:
|
|
out['completed']=False;out['failure_type']=type(e).__name__;raise
|
|
finally:
|
|
traffic.stop.set();t.join(timeout=15)
|
|
out['finished']=time.time();out['traffic']=traffic.summary();save()
|
|
if __name__=='__main__':main()
|