248 lines
15 KiB
Python
248 lines
15 KiB
Python
"""Real local multiwindow SLO experiment. Sanitized evidence, bounded faults, always restore."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import shutil
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client
|
|
BASE=os.getenv('BASE_URL','http://localhost:8086')
|
|
PROM='http://localhost:9096'
|
|
LOKI='http://localhost:3106'
|
|
TEMPO='http://localhost:3206'
|
|
def http(url):
|
|
with urllib.request.urlopen(url,timeout=30) as r:return json.load(r)
|
|
def query(expr):
|
|
return http(PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':expr}))['data']['result']
|
|
def command(args,check=True):
|
|
r=subprocess.run(['docker','compose',*args],cwd=ROOT,text=True,capture_output=True,timeout=100)
|
|
if check and r.returncode:raise RuntimeError('Compose command failed: '+' '.join(args[:3]))
|
|
return r.stdout
|
|
def save(name,obj):
|
|
(ROOT/'evidence'/name).write_text(json.dumps(obj,ensure_ascii=False,indent=2)+'\n')
|
|
def log_query(expression,limit=1000):
|
|
return http(LOKI+'/loki/api/v1/query_range?'+urllib.parse.urlencode(
|
|
{'query':expression,'start':int((time.time()-1800)*1e9),'end':int(time.time()*1e9),'limit':limit}))
|
|
def scalar(expr):
|
|
rows=query(expr)
|
|
return float(rows[0]['value'][1]) if rows else 0
|
|
class Experiment:
|
|
def __init__(self,repetitions):
|
|
self.started=time.time();self.started_mono=time.monotonic();self.aborted=None;self.phase='setup';self.samples=[];self.requests=[];self.events=[];self.uploads=[]
|
|
self.lock=threading.Lock();self.stop=threading.Event();self.repetitions=repetitions
|
|
self.client=Client(BASE)
|
|
self.alice,_,_,self.chat=self.client.bootstrap()
|
|
self.secret_strings=[self.alice['token'],self.alice['registration']['password']]
|
|
self.canary='private-body-'+uuid.uuid4().hex
|
|
self.secret_strings.append(self.canary)
|
|
def event(self,name,**extra):
|
|
data={'time':time.time(),'monotonic':time.monotonic(),'name':name,'phase':self.phase,**extra}
|
|
self.events.append(data);save('slo_events.json',self.events)
|
|
print(json.dumps(data),flush=True)
|
|
def write(self):
|
|
started=time.time();started_mono=time.monotonic();phase=self.phase
|
|
headers={'Authorization':'Bearer '+self.alice['token'],'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}
|
|
req=urllib.request.Request(BASE+'/api/v1/chats/'+self.chat['id']+'/messages',
|
|
data=json.dumps({'text':self.canary}).encode(),headers=headers,method='POST')
|
|
status=0;trace_id=''
|
|
try:
|
|
try:r=urllib.request.urlopen(req,timeout=7)
|
|
except urllib.error.HTTPError as e:r=e
|
|
with r:status=r.status;trace_id=r.headers.get('X-Trace-Id','');r.read()
|
|
except OSError:pass
|
|
row={'time':started,'phase':phase,'status':status,'seconds':time.monotonic()-started_mono,'monotonic':started_mono,'trace_id':trace_id}
|
|
with self.lock:self.requests.append(row)
|
|
def load(self):
|
|
# Open arrival: four requests/s, cap 32 in flight; count generator skips.
|
|
with ThreadPoolExecutor(max_workers=32) as pool:
|
|
pending=[];next_due=time.monotonic()
|
|
while not self.stop.is_set():
|
|
pending=[f for f in pending if not f.done()]
|
|
if len(pending)<32:pending.append(pool.submit(self.write))
|
|
else:self.event('load_generator_skipped')
|
|
next_due+=.25
|
|
self.stop.wait(max(0,next_due-time.monotonic()))
|
|
def monitor(self):
|
|
while not self.stop.is_set():
|
|
sample={'time':time.time(),'monotonic':time.monotonic(),'phase':self.phase,'windows_c_free_bytes':shutil.disk_usage('/mnt/c').free}
|
|
if sample['windows_c_free_bytes']<2*1024**3:
|
|
self.aborted='C drive free space below 2 GiB';self.event('resource_abort',reason=self.aborted);self.stop.set()
|
|
try:
|
|
sample['alerts']=http(PROM+'/api/v1/alerts')['data']['alerts']
|
|
sample['metrics']={expr:query(expr) for expr in [
|
|
'max(queue_depth)','max(queue_oldest_age_seconds)','sum(db_pool_waiting)',
|
|
'slo:api_error_ratio:30s','slo:api_error_ratio:2m','slo:api_error_ratio:5m',
|
|
'slo:ready_bad_ratio:30s',
|
|
'sum(otelcol_exporter_send_failed_spans_total)','sum(otelcol_exporter_enqueue_failed_spans_total)',
|
|
'sum(otelcol_receiver_accepted_spans_total)','sum(otelcol_exporter_sent_spans_total)','max(otelcol_exporter_queue_size)']}
|
|
sample['resources']=command(['stats','--no-stream','--format','{{json .}}'])
|
|
except Exception as e:sample['error']=type(e).__name__
|
|
self.samples.append(sample);save('slo_samples.json',self.samples)
|
|
with self.lock:save('slo_requests.json',self.requests)
|
|
save('slo_result.json',self.summary('RUNNING'))
|
|
self.stop.wait(3)
|
|
def guard(self):
|
|
if self.aborted:raise RuntimeError(self.aborted)
|
|
def hold(self,phase,seconds):
|
|
self.phase=phase;self.event('phase_started',duration=seconds)
|
|
end=time.monotonic()+seconds
|
|
while time.monotonic()<end:
|
|
self.guard();time.sleep(max(0,min(1,end-time.monotonic())))
|
|
def upload(self):
|
|
r=self.client.call('POST','/api/v1/chats/'+self.chat['id']+'/attachments?filename=synthetic.png',
|
|
202,actor=self.alice,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),
|
|
headers={'Content-Type':'image/png'},schema='Attachment')
|
|
row={'id':r['json']['id'],'trace_id':r['headers'].get('X-Trace-Id'),'accepted_at':time.time(),'phase':self.phase}
|
|
self.uploads.append(row);save('slo_uploads.json',self.uploads);return row
|
|
def ready(self,upload,timeout=90):
|
|
end=time.monotonic()+timeout
|
|
while time.monotonic()<end:
|
|
r=self.client.call('GET','/api/v1/attachments/'+upload['id'],actor=self.alice,schema='Attachment')['json']
|
|
if r['status']=='ready':
|
|
original=self.client.call('GET','/api/v1/attachments/'+upload['id']+'/content',actor=self.alice)['bytes']
|
|
assert hashlib.sha256(original).hexdigest()==r['sha256']
|
|
assert len(r['variants'])==3
|
|
upload['ready_at']=time.time();upload['elapsed_to_observed_ready']=upload['ready_at']-upload['accepted_at']
|
|
return
|
|
if r['status']=='failed':raise AssertionError('Attachment failed')
|
|
time.sleep(.5)
|
|
raise AssertionError('Attachment ready timeout')
|
|
def db_fault(self,seconds,phase):
|
|
self.guard();self.phase=phase
|
|
# Real table lock. Health SELECT 1 stays ready; SQL statement timeout -> API 503.
|
|
process=subprocess.Popen(['docker','compose','exec','-T','db','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',
|
|
f"BEGIN; SET LOCAL application_name='lab6-bounded-fault'; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;"],
|
|
cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
self.event('db_lock_requested',seconds=seconds)
|
|
try:
|
|
time.sleep(.5)
|
|
lock=command(['exec','-T','db','psql','-U','messenger','-d','messenger','-tAc',
|
|
"SELECT count(*) FROM pg_locks WHERE relation='messages'::regclass AND mode='AccessExclusiveLock' AND granted"])
|
|
assert int(lock.strip())==1
|
|
self.event('db_lock_acquired')
|
|
end=time.monotonic()+seconds+15
|
|
while process.poll() is None and time.monotonic()<end:
|
|
self.guard();time.sleep(.5)
|
|
assert process.poll()==0
|
|
finally:
|
|
if process.poll() is None:process.terminate()
|
|
self.event('db_lock_released')
|
|
def observations(self,name):
|
|
data={'phase':self.phase,'api_logs':log_query('{service_name="messenger-api"}'),
|
|
'worker_logs':log_query('{service_name="messenger-worker"}')}
|
|
errors=[r for r in self.requests if r['status']>=500 and r['trace_id']]
|
|
ids=[u['trace_id'] for u in self.uploads if u['trace_id']]+[r['trace_id'] for r in errors[-3:]]
|
|
data['traces']={}
|
|
for tid in set(ids):
|
|
try:data['traces'][tid]=http(TEMPO+'/api/traces/'+tid)
|
|
except Exception as e:data['traces'][tid]={'unavailable':type(e).__name__}
|
|
raw=json.dumps(data)
|
|
assert all(secret not in raw for secret in self.secret_strings),'Telemetry leaked synthetic secret/body'
|
|
save(name,data)
|
|
return data
|
|
def run(self):
|
|
def interrupted(signum,frame):
|
|
self.aborted='Interrupted by signal '+str(signum)
|
|
raise KeyboardInterrupt(self.aborted)
|
|
signal.signal(signal.SIGTERM,interrupted)
|
|
signal.signal(signal.SIGINT,interrupted)
|
|
threads=[threading.Thread(target=self.load),threading.Thread(target=self.monitor)]
|
|
for t in threads:t.start()
|
|
status='FAILED'
|
|
try:
|
|
self.hold('baseline',120)
|
|
# Healthy upload gives baseline time-to-ready.
|
|
healthy=self.upload();self.ready(healthy)
|
|
for repeat in range(1,self.repetitions+1):
|
|
prefix=str(repeat)+'_'
|
|
self.db_fault(5,prefix+'short_spike')
|
|
self.hold(prefix+'short_recovery',70)
|
|
self.phase=prefix+'worker_stopped'
|
|
command(['stop','worker']);self.event('worker_stopped')
|
|
queued=[self.upload() for _ in range(4)]
|
|
self.hold(prefix+'worker_stopped',40)
|
|
# Nominal S3 stop; Compose may start dependencies. Real retry proof: retry_trace.py.
|
|
command(['stop','s3']);command(['start','worker']);self.event('worker_started_s3_down')
|
|
self.hold(prefix+'s3_retry',6)
|
|
command(['start','s3']);self.event('s3_restored')
|
|
self.phase=prefix+'worker_recovery'
|
|
for u in queued:self.ready(u)
|
|
self.event('worker_queue_drained')
|
|
self.hold(prefix+'worker_recovery',50)
|
|
self.observations('signals_worker_'+str(repeat)+'.json')
|
|
self.db_fault(65,prefix+'db_degraded')
|
|
self.hold(prefix+'db_recovery',85)
|
|
self.observations('signals_db_'+str(repeat)+'.json')
|
|
self.phase='backend_down'
|
|
self.before_backend={q:query(q) for q in ['sum(otelcol_exporter_send_failed_spans_total)','sum(otelcol_exporter_sent_spans_total)','sum(otelcol_receiver_accepted_spans_total)']}
|
|
command(['stop','tempo']);self.event('tempo_stopped')
|
|
backend_upload=self.upload();self.ready(backend_upload)
|
|
self.hold('backend_down',35)
|
|
self.after_backend={q:query(q) for q in self.before_backend}
|
|
command(['start','tempo']);self.event('tempo_started')
|
|
self.hold('backend_recovery',25)
|
|
save('backend_outage.json',{'before':self.before_backend,'during':self.after_backend,'after_recovery':{q:query(q) for q in self.before_backend},'upload':backend_upload})
|
|
self.observations('signals_final.json')
|
|
status='PASS'
|
|
finally:
|
|
self.stop.set()
|
|
for t in threads:t.join(timeout=15)
|
|
if self.aborted:command(['stop'],check=False)
|
|
else:
|
|
for service in ['db','worker','s3','tempo']:command(['start',service],check=False)
|
|
save('slo_requests.json',self.requests);save('slo_uploads.json',self.uploads)
|
|
save('slo_result.json',self.summary(status))
|
|
def summary(self,status):
|
|
phases={}
|
|
for phase in sorted({r['phase'] for r in self.requests}):
|
|
rows=[r for r in self.requests if r['phase']==phase];success=sorted(r['seconds'] for r in rows if r['status']==201)
|
|
duration=max(r['monotonic']+r['seconds'] for r in rows)-min(r['monotonic'] for r in rows)
|
|
phases[phase]={'requests':len(rows),'successes':len(success),'errors':len(rows)-len(success),
|
|
'successful_rps':len(success)/duration,'successful_latency_ms':{
|
|
name:success[max(0,math.ceil(q*len(success))-1)]*1000 if success else None for name,q in [('p50',.5),('p95',.95),('p99',.99)]}}
|
|
detections=[]
|
|
incidents=[e for e in self.events if e['name'] in ['worker_stopped','db_lock_acquired']]
|
|
for index,event in enumerate(incidents):
|
|
end=incidents[index+1]['monotonic'] if index+1<len(incidents) else float('inf')
|
|
names=['QueueLagGrowing','ReadyErrorBudgetBurn'] if event['name']=='worker_stopped' else ['APIHighErrorRate','APIErrorBudgetBurnFast','APIErrorBudgetBurnSlow']
|
|
recovery_name='worker_queue_drained' if event['name']=='worker_stopped' else 'db_lock_released'
|
|
recovery=next((e for e in self.events if event['monotonic']<e['monotonic']<end and e['name']==recovery_name),None)
|
|
for name in names:
|
|
samples=[s for s in self.samples if event['monotonic']<=s['monotonic']<end and 'error' not in s]
|
|
firing=[s for s in samples if any(a['labels']['alertname']==name and a['state']=='firing' for a in s.get('alerts',[]))]
|
|
first=firing[0] if firing else None
|
|
later=[s for s in samples if first and s['monotonic']>first['monotonic'] and not any(a['labels']['alertname']==name and a['state']=='firing' for a in s.get('alerts',[]))]
|
|
resolved=later[0] if later else None
|
|
detections.append({'incident':event['phase'],'alert':name,
|
|
'detection_seconds':first['monotonic']-event['monotonic'] if first else None,
|
|
'resolved_time':resolved['time'] if resolved else None,
|
|
'recovery_seconds':resolved['monotonic']-recovery['monotonic'] if resolved and recovery else None})
|
|
false_pages=[]
|
|
for s in self.samples:
|
|
if s['phase']=='baseline' or 'short_' in s['phase']:
|
|
for a in s.get('alerts',[]):
|
|
if a['state']=='firing' and a['labels']['alertname'] in ['APIErrorBudgetBurnFast','APIErrorBudgetBurnSlow']:
|
|
false_pages.append({'phase':s['phase'],'alert':a['labels']['alertname'],'time':s['time']})
|
|
return {'status':status,'repetitions':self.repetitions,'elapsed_seconds':time.monotonic()-self.started_mono,'aborted':self.aborted,
|
|
'workload':'4 arrival/s, max 32 in-flight, no retry; synthetic messages; 4 queued PNG per worker incident',
|
|
'phases':phases,'detections':detections,'false_page_samples':false_pages,
|
|
'hypothesis':'A five-second DB lock does not page multiwindow burn; sustained DB lock and worker stop are detected and resolve.',
|
|
'note':'PASS means experiment executed; criterion assertions are checked separately by audit.'}
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('--repetitions',type=int,default=2)
|
|
args=p.parse_args();assert 1<=args.repetitions<=3
|
|
Experiment(args.repetitions).run()
|