Files
mtusi2026lab6/scripts/observe_experiment.py
T
2026-10-30 21:38:40 +03:00

248 lines
15 KiB
Python

"""Real local multiwindow SLO experiment. Sanitized evidence, bounded faults, always restore."""
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import argparse
import hashlib
import json
import math
import os
import subprocess
import shutil
import signal
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client
BASE=os.getenv('BASE_URL','http://localhost:8086')
PROM='http://localhost:9096'
LOKI='http://localhost:3106'
TEMPO='http://localhost:3206'
def http(url):
with urllib.request.urlopen(url,timeout=30) as r:return json.load(r)
def query(expr):
return http(PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':expr}))['data']['result']
def command(args,check=True):
r=subprocess.run(['docker','compose',*args],cwd=ROOT,text=True,capture_output=True,timeout=100)
if check and r.returncode:raise RuntimeError('Compose command failed: '+' '.join(args[:3]))
return r.stdout
def save(name,obj):
(ROOT/'evidence'/name).write_text(json.dumps(obj,ensure_ascii=False,indent=2)+'\n')
def log_query(expression,limit=1000):
return http(LOKI+'/loki/api/v1/query_range?'+urllib.parse.urlencode(
{'query':expression,'start':int((time.time()-1800)*1e9),'end':int(time.time()*1e9),'limit':limit}))
def scalar(expr):
rows=query(expr)
return float(rows[0]['value'][1]) if rows else 0
class Experiment:
def __init__(self,repetitions):
self.started=time.time();self.started_mono=time.monotonic();self.aborted=None;self.phase='setup';self.samples=[];self.requests=[];self.events=[];self.uploads=[]
self.lock=threading.Lock();self.stop=threading.Event();self.repetitions=repetitions
self.client=Client(BASE)
self.alice,_,_,self.chat=self.client.bootstrap()
self.secret_strings=[self.alice['token'],self.alice['registration']['password']]
self.canary='private-body-'+uuid.uuid4().hex
self.secret_strings.append(self.canary)
def event(self,name,**extra):
data={'time':time.time(),'monotonic':time.monotonic(),'name':name,'phase':self.phase,**extra}
self.events.append(data);save('slo_events.json',self.events)
print(json.dumps(data),flush=True)
def write(self):
started=time.time();started_mono=time.monotonic();phase=self.phase
headers={'Authorization':'Bearer '+self.alice['token'],'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}
req=urllib.request.Request(BASE+'/api/v1/chats/'+self.chat['id']+'/messages',
data=json.dumps({'text':self.canary}).encode(),headers=headers,method='POST')
status=0;trace_id=''
try:
try:r=urllib.request.urlopen(req,timeout=7)
except urllib.error.HTTPError as e:r=e
with r:status=r.status;trace_id=r.headers.get('X-Trace-Id','');r.read()
except OSError:pass
row={'time':started,'phase':phase,'status':status,'seconds':time.monotonic()-started_mono,'monotonic':started_mono,'trace_id':trace_id}
with self.lock:self.requests.append(row)
def load(self):
# Open arrival: four requests/s, cap 32 in flight; count generator skips.
with ThreadPoolExecutor(max_workers=32) as pool:
pending=[];next_due=time.monotonic()
while not self.stop.is_set():
pending=[f for f in pending if not f.done()]
if len(pending)<32:pending.append(pool.submit(self.write))
else:self.event('load_generator_skipped')
next_due+=.25
self.stop.wait(max(0,next_due-time.monotonic()))
def monitor(self):
while not self.stop.is_set():
sample={'time':time.time(),'monotonic':time.monotonic(),'phase':self.phase,'windows_c_free_bytes':shutil.disk_usage('/mnt/c').free}
if sample['windows_c_free_bytes']<2*1024**3:
self.aborted='C drive free space below 2 GiB';self.event('resource_abort',reason=self.aborted);self.stop.set()
try:
sample['alerts']=http(PROM+'/api/v1/alerts')['data']['alerts']
sample['metrics']={expr:query(expr) for expr in [
'max(queue_depth)','max(queue_oldest_age_seconds)','sum(db_pool_waiting)',
'slo:api_error_ratio:30s','slo:api_error_ratio:2m','slo:api_error_ratio:5m',
'slo:ready_bad_ratio:30s',
'sum(otelcol_exporter_send_failed_spans_total)','sum(otelcol_exporter_enqueue_failed_spans_total)',
'sum(otelcol_receiver_accepted_spans_total)','sum(otelcol_exporter_sent_spans_total)','max(otelcol_exporter_queue_size)']}
sample['resources']=command(['stats','--no-stream','--format','{{json .}}'])
except Exception as e:sample['error']=type(e).__name__
self.samples.append(sample);save('slo_samples.json',self.samples)
with self.lock:save('slo_requests.json',self.requests)
save('slo_result.json',self.summary('RUNNING'))
self.stop.wait(3)
def guard(self):
if self.aborted:raise RuntimeError(self.aborted)
def hold(self,phase,seconds):
self.phase=phase;self.event('phase_started',duration=seconds)
end=time.monotonic()+seconds
while time.monotonic()<end:
self.guard();time.sleep(max(0,min(1,end-time.monotonic())))
def upload(self):
r=self.client.call('POST','/api/v1/chats/'+self.chat['id']+'/attachments?filename=synthetic.png',
202,actor=self.alice,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),
headers={'Content-Type':'image/png'},schema='Attachment')
row={'id':r['json']['id'],'trace_id':r['headers'].get('X-Trace-Id'),'accepted_at':time.time(),'phase':self.phase}
self.uploads.append(row);save('slo_uploads.json',self.uploads);return row
def ready(self,upload,timeout=90):
end=time.monotonic()+timeout
while time.monotonic()<end:
r=self.client.call('GET','/api/v1/attachments/'+upload['id'],actor=self.alice,schema='Attachment')['json']
if r['status']=='ready':
original=self.client.call('GET','/api/v1/attachments/'+upload['id']+'/content',actor=self.alice)['bytes']
assert hashlib.sha256(original).hexdigest()==r['sha256']
assert len(r['variants'])==3
upload['ready_at']=time.time();upload['elapsed_to_observed_ready']=upload['ready_at']-upload['accepted_at']
return
if r['status']=='failed':raise AssertionError('Attachment failed')
time.sleep(.5)
raise AssertionError('Attachment ready timeout')
def db_fault(self,seconds,phase):
self.guard();self.phase=phase
# Real table lock. Health SELECT 1 stays ready; SQL statement timeout -> API 503.
process=subprocess.Popen(['docker','compose','exec','-T','db','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',
f"BEGIN; SET LOCAL application_name='lab6-bounded-fault'; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;"],
cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
self.event('db_lock_requested',seconds=seconds)
try:
time.sleep(.5)
lock=command(['exec','-T','db','psql','-U','messenger','-d','messenger','-tAc',
"SELECT count(*) FROM pg_locks WHERE relation='messages'::regclass AND mode='AccessExclusiveLock' AND granted"])
assert int(lock.strip())==1
self.event('db_lock_acquired')
end=time.monotonic()+seconds+15
while process.poll() is None and time.monotonic()<end:
self.guard();time.sleep(.5)
assert process.poll()==0
finally:
if process.poll() is None:process.terminate()
self.event('db_lock_released')
def observations(self,name):
data={'phase':self.phase,'api_logs':log_query('{service_name="messenger-api"}'),
'worker_logs':log_query('{service_name="messenger-worker"}')}
errors=[r for r in self.requests if r['status']>=500 and r['trace_id']]
ids=[u['trace_id'] for u in self.uploads if u['trace_id']]+[r['trace_id'] for r in errors[-3:]]
data['traces']={}
for tid in set(ids):
try:data['traces'][tid]=http(TEMPO+'/api/traces/'+tid)
except Exception as e:data['traces'][tid]={'unavailable':type(e).__name__}
raw=json.dumps(data)
assert all(secret not in raw for secret in self.secret_strings),'Telemetry leaked synthetic secret/body'
save(name,data)
return data
def run(self):
def interrupted(signum,frame):
self.aborted='Interrupted by signal '+str(signum)
raise KeyboardInterrupt(self.aborted)
signal.signal(signal.SIGTERM,interrupted)
signal.signal(signal.SIGINT,interrupted)
threads=[threading.Thread(target=self.load),threading.Thread(target=self.monitor)]
for t in threads:t.start()
status='FAILED'
try:
self.hold('baseline',120)
# Healthy upload gives baseline time-to-ready.
healthy=self.upload();self.ready(healthy)
for repeat in range(1,self.repetitions+1):
prefix=str(repeat)+'_'
self.db_fault(5,prefix+'short_spike')
self.hold(prefix+'short_recovery',70)
self.phase=prefix+'worker_stopped'
command(['stop','worker']);self.event('worker_stopped')
queued=[self.upload() for _ in range(4)]
self.hold(prefix+'worker_stopped',40)
# Nominal S3 stop; Compose may start dependencies. Real retry proof: retry_trace.py.
command(['stop','s3']);command(['start','worker']);self.event('worker_started_s3_down')
self.hold(prefix+'s3_retry',6)
command(['start','s3']);self.event('s3_restored')
self.phase=prefix+'worker_recovery'
for u in queued:self.ready(u)
self.event('worker_queue_drained')
self.hold(prefix+'worker_recovery',50)
self.observations('signals_worker_'+str(repeat)+'.json')
self.db_fault(65,prefix+'db_degraded')
self.hold(prefix+'db_recovery',85)
self.observations('signals_db_'+str(repeat)+'.json')
self.phase='backend_down'
self.before_backend={q:query(q) for q in ['sum(otelcol_exporter_send_failed_spans_total)','sum(otelcol_exporter_sent_spans_total)','sum(otelcol_receiver_accepted_spans_total)']}
command(['stop','tempo']);self.event('tempo_stopped')
backend_upload=self.upload();self.ready(backend_upload)
self.hold('backend_down',35)
self.after_backend={q:query(q) for q in self.before_backend}
command(['start','tempo']);self.event('tempo_started')
self.hold('backend_recovery',25)
save('backend_outage.json',{'before':self.before_backend,'during':self.after_backend,'after_recovery':{q:query(q) for q in self.before_backend},'upload':backend_upload})
self.observations('signals_final.json')
status='PASS'
finally:
self.stop.set()
for t in threads:t.join(timeout=15)
if self.aborted:command(['stop'],check=False)
else:
for service in ['db','worker','s3','tempo']:command(['start',service],check=False)
save('slo_requests.json',self.requests);save('slo_uploads.json',self.uploads)
save('slo_result.json',self.summary(status))
def summary(self,status):
phases={}
for phase in sorted({r['phase'] for r in self.requests}):
rows=[r for r in self.requests if r['phase']==phase];success=sorted(r['seconds'] for r in rows if r['status']==201)
duration=max(r['monotonic']+r['seconds'] for r in rows)-min(r['monotonic'] for r in rows)
phases[phase]={'requests':len(rows),'successes':len(success),'errors':len(rows)-len(success),
'successful_rps':len(success)/duration,'successful_latency_ms':{
name:success[max(0,math.ceil(q*len(success))-1)]*1000 if success else None for name,q in [('p50',.5),('p95',.95),('p99',.99)]}}
detections=[]
incidents=[e for e in self.events if e['name'] in ['worker_stopped','db_lock_acquired']]
for index,event in enumerate(incidents):
end=incidents[index+1]['monotonic'] if index+1<len(incidents) else float('inf')
names=['QueueLagGrowing','ReadyErrorBudgetBurn'] if event['name']=='worker_stopped' else ['APIHighErrorRate','APIErrorBudgetBurnFast','APIErrorBudgetBurnSlow']
recovery_name='worker_queue_drained' if event['name']=='worker_stopped' else 'db_lock_released'
recovery=next((e for e in self.events if event['monotonic']<e['monotonic']<end and e['name']==recovery_name),None)
for name in names:
samples=[s for s in self.samples if event['monotonic']<=s['monotonic']<end and 'error' not in s]
firing=[s for s in samples if any(a['labels']['alertname']==name and a['state']=='firing' for a in s.get('alerts',[]))]
first=firing[0] if firing else None
later=[s for s in samples if first and s['monotonic']>first['monotonic'] and not any(a['labels']['alertname']==name and a['state']=='firing' for a in s.get('alerts',[]))]
resolved=later[0] if later else None
detections.append({'incident':event['phase'],'alert':name,
'detection_seconds':first['monotonic']-event['monotonic'] if first else None,
'resolved_time':resolved['time'] if resolved else None,
'recovery_seconds':resolved['monotonic']-recovery['monotonic'] if resolved and recovery else None})
false_pages=[]
for s in self.samples:
if s['phase']=='baseline' or 'short_' in s['phase']:
for a in s.get('alerts',[]):
if a['state']=='firing' and a['labels']['alertname'] in ['APIErrorBudgetBurnFast','APIErrorBudgetBurnSlow']:
false_pages.append({'phase':s['phase'],'alert':a['labels']['alertname'],'time':s['time']})
return {'status':status,'repetitions':self.repetitions,'elapsed_seconds':time.monotonic()-self.started_mono,'aborted':self.aborted,
'workload':'4 arrival/s, max 32 in-flight, no retry; synthetic messages; 4 queued PNG per worker incident',
'phases':phases,'detections':detections,'false_page_samples':false_pages,
'hypothesis':'A five-second DB lock does not page multiwindow burn; sustained DB lock and worker stop are detected and resolve.',
'note':'PASS means experiment executed; criterion assertions are checked separately by audit.'}
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--repetitions',type=int,default=2)
args=p.parse_args();assert 1<=args.repetitions<=3
Experiment(args.repetitions).run()