Files
mtusi2026lab6/scripts/incidents.py
T

161 lines
10 KiB
Python

"""Bounded, repeatable SLO and telemetry-backend fault experiments (local stack only)."""
import concurrent.futures
import json
import math
import subprocess
import threading
import time
import uuid
import httpx
from stack import ROOT,docker,ready
from verify_telemetry import bootstrap,upload,wait_ready,query,sql,trace_spans
OUT=ROOT/'evidence/incidents'; OUT.mkdir(parents=True,exist_ok=True)
stop=threading.Event(); requests=[]; lock=threading.Lock(); results={'phases':[]}
def traffic(headers,chat):
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=10) as client:
while not stop.is_set():
trace=uuid.uuid4().hex; start=time.time()
try:
r=client.post(f'/api/v1/chats/{chat}/messages',headers={**headers,'Idempotency-Key':uuid.uuid4().hex,
'traceparent':f'00-{trace}-{uuid.uuid4().hex[:16]}-01'},json={'text':'synthetic SLO probe'})
status=r.status_code
except httpx.HTTPError: status=0
with lock: requests.append({'at':start,'seconds':time.time()-start,'status':status,'trace_id':trace})
stop.wait(.2)
def alerts():
r=httpx.get('http://127.0.0.1:9090/api/v1/alerts',timeout=10); r.raise_for_status()
return {a['labels']['alertname']:a['state'] for a in r.json()['data']['alerts']}
def phase(name,duration):
print('Phase '+name,flush=True)
start=time.time(); phase={'name':name,'start':start,'samples':[]}
results['phases'].append(phase)
until=time.monotonic()+duration
while True:
sample={'at':time.time(),'alerts':alerts()}
for label,expr in {
'queue_depth':'sum(queue_depth)','queue_age':'max(queue_oldest_age_seconds)',
'api_error_ratio':'(sum(rate(http_server_requests_total{status_class="5xx"}[20s])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[20s])),0.001)',
'api_p95':'histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))',
'db_pool_used':'sum(db_pool_used)','db_pool_waiting':'sum(db_pool_waiting)',
'worker_busy':'sum(worker_busy)','collector_queue':'otelcol_exporter_queue_size',
'collector_capacity':'otelcol_exporter_queue_capacity',
'collector_failed_spans':'otelcol_exporter_send_failed_spans_total',
'collector_failed_logs':'otelcol_exporter_send_failed_log_records_total',
'collector_enqueue_failed_spans':'otelcol_exporter_enqueue_failed_spans_total',
'collector_enqueue_failed_logs':'otelcol_exporter_enqueue_failed_log_records_total',
'collector_refused_spans':'otelcol_receiver_refused_spans_total',
'collector_refused_logs':'otelcol_receiver_refused_log_records_total',
'collector_received_spans':'otelcol_receiver_accepted_spans_total',
'collector_sent_spans':'otelcol_exporter_sent_spans_total',
'collector_received_logs':'otelcol_receiver_accepted_log_records_total',
'collector_sent_logs':'otelcol_exporter_sent_log_records_total',
'collector_rss':'otelcol_process_memory_rss_bytes','api_worker_rss':'process_rss_bytes'}.items():
sample[label]=query(expr)
phase['samples'].append(sample)
if time.monotonic()>=until: break
time.sleep(min(5,max(0,until-time.monotonic())))
phase['end']=time.time()
with lock: rows=[r for r in requests if start<=r['at']<phase['end']]
latencies=sorted(r['seconds'] for r in rows if r['status']==201)
phase['requests']={'total':len(rows),'success':len(latencies),'errors':len(rows)-len(latencies),
'successful_rps':len(latencies)/(phase['end']-start),
'successful_latency_ms':{f'p{p}':1000*latencies[max(0,math.ceil(p/100*len(latencies))-1)] if latencies else None for p in (50,95,99)}}
(OUT/'timeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
return phase
def fired(phase,name): return [s['at']-phase['start'] for s in phase['samples'] if s['alerts'].get(name)=='firing']
def db_lock(seconds):
proc=subprocess.Popen(['docker','compose','exec','-T','db','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',
f'BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({int(seconds)}); COMMIT;'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
deadline=time.monotonic()+10
while time.monotonic()<deadline:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_class c ON c.oid=l.relation WHERE c.relname='sessions' AND l.mode='AccessExclusiveLock' AND l.granted")=='1': return proc
time.sleep(.2)
raise AssertionError('DB fault did not acquire lock')
def main():
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
ready(c); headers,chat,_=bootstrap(c)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
futures=[pool.submit(traffic,headers,chat) for _ in range(4)]
try:
quiet=phase('preflight_settle',120)
assert not any(s=='firing' for s in quiet['samples'][-1]['alerts'].values()),'Prior incident has not cleared'
phase('baseline',45)
proc=db_lock(5)
short=phase('db_burst_5s',45); proc.wait(timeout=10)
assert not fired(short,'APIAvailabilityBurn'),'Short spike generated false alert'
phase('settle',35)
proc=db_lock(70)
sustained=phase('db_sustained_70s',75); proc.wait(timeout=10)
assert fired(sustained,'APIAvailabilityBurn'),'Sustained DB failure not detected'
recovered=phase('db_recovery',70)
assert recovered['samples'][-1]['alerts'].get('APIAvailabilityBurn')!='firing'
errors=[r for r in requests if sustained['start']<=r['at']<sustained['end'] and r['status']==503]
assert errors,'No injected DB error was observed'
trace_id=errors[len(errors)//2]['trace_id']
response=httpx.get('http://127.0.0.1:3200/api/traces/'+trace_id,headers={'Accept':'application/json'},timeout=15)
response.raise_for_status(); trace=response.json()
db_spans=[s for s in trace_spans(trace) if s['name']=='db.session.authenticate']
assert db_spans and any(s.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR') for s in db_spans)
(OUT/'db-error-trace.json').write_text(json.dumps(trace,indent=2)+'\n',encoding='utf-8')
docker('compose','stop','worker')
try:
aids=[upload(c,headers,chat).json()['id'] for _ in range(8)]
queued=phase('worker_stopped',85)
assert fired(queued,'QueueLag') and fired(queued,'AttachmentReadyBurn')
finally: docker('compose','start','worker')
started=time.time()
for aid in aids: wait_ready(c,headers,aid)
results['queue_seconds_to_drain']=time.time()-started
queue_recovered=phase('worker_recovery',110)
assert not {'QueueLag','AttachmentReadyBurn'} & {n for n,s in queue_recovered['samples'][-1]['alerts'].items() if s=='firing'}
docker('compose','stop','-t','3','loki','tempo')
try:
backend=phase('telemetry_backends_stopped',80)
assert backend['requests']['success']>0 and backend['requests']['errors']==0
finally: docker('compose','start','loki','tempo')
phase('telemetry_recovery',45)
lines=[line for line in docker('compose','logs','--no-color','otel-collector').splitlines()
if 'Exporting failed. Dropping data.' in line or 'sending_queue is full' in line]
(OUT/'collector-drops.txt').write_text('\n'.join(lines)+'\n',encoding='utf-8')
dropped={}
for line in lines:
if '"dropped_items"' in line:
entry=json.loads(line[line.index('{'):]); signal=entry['otelcol.signal']
dropped[signal]=dropped.get(signal,0)+entry['dropped_items']
results['collector_explicit_retry_drops']=dropped
for sample in backend['samples']:
caps={row['metric'].get('exporter'):float(row['value'][1]) for row in sample['collector_capacity']}
for row in sample['collector_queue']:
assert float(row['value'][1])<=caps[row['metric'].get('exporter')]
results['summary']={'short_burst_false_firings':len(fired(short,'APIAvailabilityBurn')),
'api_detection_seconds':fired(sustained,'APIAvailabilityBurn')[0],
'queue_lag_detection_seconds':fired(queued,'QueueLag')[0],
'ready_slo_detection_seconds':fired(queued,'AttachmentReadyBurn')[0],
'db_error_trace_id':trace_id,'backend_outage_business_requests':backend['requests'],'passed':True}
results['passed']=True
finally:
stop.set()
for f in futures: f.result(timeout=15)
(OUT/'timeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
(OUT/'requests.json').write_text(json.dumps(requests,indent=2)+'\n',encoding='utf-8')
# After traffic stops, allow SDK/batch/scrape to flush. There are no health
# spans. Compare record counters after exporter queues have drained.
time.sleep(15)
def total(expr): return sum(float(r['value'][1]) for r in query(expr))
assert total('otelcol_exporter_queue_size')==0
results['quiescent_collector']={}
for signal in ('spans','log_records'):
accepted=total('otelcol_receiver_accepted_'+signal+'_total')
sent=total('otelcol_exporter_sent_'+signal+'_total')
results['quiescent_collector'][signal]={'accepted':accepted,'sent':sent,'not_exported':accepted-sent}
(OUT/'timeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
print(json.dumps(results['summary'],indent=2),flush=True)
if __name__=='__main__': main()