161 lines
10 KiB
Python
161 lines
10 KiB
Python
"""Bounded, repeatable SLO and telemetry-backend fault experiments (local stack only)."""
|
|
import concurrent.futures
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import httpx
|
|
from stack import ROOT,docker,ready
|
|
from verify_telemetry import bootstrap,upload,wait_ready,query,sql,trace_spans
|
|
|
|
OUT=ROOT/'evidence/incidents'; OUT.mkdir(parents=True,exist_ok=True)
|
|
stop=threading.Event(); requests=[]; lock=threading.Lock(); results={'phases':[]}
|
|
|
|
def traffic(headers,chat):
|
|
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=10) as client:
|
|
while not stop.is_set():
|
|
trace=uuid.uuid4().hex; start=time.time()
|
|
try:
|
|
r=client.post(f'/api/v1/chats/{chat}/messages',headers={**headers,'Idempotency-Key':uuid.uuid4().hex,
|
|
'traceparent':f'00-{trace}-{uuid.uuid4().hex[:16]}-01'},json={'text':'synthetic SLO probe'})
|
|
status=r.status_code
|
|
except httpx.HTTPError: status=0
|
|
with lock: requests.append({'at':start,'seconds':time.time()-start,'status':status,'trace_id':trace})
|
|
stop.wait(.2)
|
|
|
|
def alerts():
|
|
r=httpx.get('http://127.0.0.1:9090/api/v1/alerts',timeout=10); r.raise_for_status()
|
|
return {a['labels']['alertname']:a['state'] for a in r.json()['data']['alerts']}
|
|
|
|
def phase(name,duration):
|
|
print('Phase '+name,flush=True)
|
|
start=time.time(); phase={'name':name,'start':start,'samples':[]}
|
|
results['phases'].append(phase)
|
|
until=time.monotonic()+duration
|
|
while True:
|
|
sample={'at':time.time(),'alerts':alerts()}
|
|
for label,expr in {
|
|
'queue_depth':'sum(queue_depth)','queue_age':'max(queue_oldest_age_seconds)',
|
|
'api_error_ratio':'(sum(rate(http_server_requests_total{status_class="5xx"}[20s])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[20s])),0.001)',
|
|
'api_p95':'histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))',
|
|
'db_pool_used':'sum(db_pool_used)','db_pool_waiting':'sum(db_pool_waiting)',
|
|
'worker_busy':'sum(worker_busy)','collector_queue':'otelcol_exporter_queue_size',
|
|
'collector_capacity':'otelcol_exporter_queue_capacity',
|
|
'collector_failed_spans':'otelcol_exporter_send_failed_spans_total',
|
|
'collector_failed_logs':'otelcol_exporter_send_failed_log_records_total',
|
|
'collector_enqueue_failed_spans':'otelcol_exporter_enqueue_failed_spans_total',
|
|
'collector_enqueue_failed_logs':'otelcol_exporter_enqueue_failed_log_records_total',
|
|
'collector_refused_spans':'otelcol_receiver_refused_spans_total',
|
|
'collector_refused_logs':'otelcol_receiver_refused_log_records_total',
|
|
'collector_received_spans':'otelcol_receiver_accepted_spans_total',
|
|
'collector_sent_spans':'otelcol_exporter_sent_spans_total',
|
|
'collector_received_logs':'otelcol_receiver_accepted_log_records_total',
|
|
'collector_sent_logs':'otelcol_exporter_sent_log_records_total',
|
|
'collector_rss':'otelcol_process_memory_rss_bytes','api_worker_rss':'process_rss_bytes'}.items():
|
|
sample[label]=query(expr)
|
|
phase['samples'].append(sample)
|
|
if time.monotonic()>=until: break
|
|
time.sleep(min(5,max(0,until-time.monotonic())))
|
|
phase['end']=time.time()
|
|
with lock: rows=[r for r in requests if start<=r['at']<phase['end']]
|
|
latencies=sorted(r['seconds'] for r in rows if r['status']==201)
|
|
phase['requests']={'total':len(rows),'success':len(latencies),'errors':len(rows)-len(latencies),
|
|
'successful_rps':len(latencies)/(phase['end']-start),
|
|
'successful_latency_ms':{f'p{p}':1000*latencies[max(0,math.ceil(p/100*len(latencies))-1)] if latencies else None for p in (50,95,99)}}
|
|
(OUT/'timeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
|
|
return phase
|
|
|
|
def fired(phase,name): return [s['at']-phase['start'] for s in phase['samples'] if s['alerts'].get(name)=='firing']
|
|
def db_lock(seconds):
|
|
proc=subprocess.Popen(['docker','compose','exec','-T','db','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',
|
|
f'BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({int(seconds)}); COMMIT;'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
deadline=time.monotonic()+10
|
|
while time.monotonic()<deadline:
|
|
if sql("SELECT count(*) FROM pg_locks l JOIN pg_class c ON c.oid=l.relation WHERE c.relname='sessions' AND l.mode='AccessExclusiveLock' AND l.granted")=='1': return proc
|
|
time.sleep(.2)
|
|
raise AssertionError('DB fault did not acquire lock')
|
|
|
|
def main():
|
|
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
|
|
ready(c); headers,chat,_=bootstrap(c)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
futures=[pool.submit(traffic,headers,chat) for _ in range(4)]
|
|
try:
|
|
quiet=phase('preflight_settle',120)
|
|
assert not any(s=='firing' for s in quiet['samples'][-1]['alerts'].values()),'Prior incident has not cleared'
|
|
phase('baseline',45)
|
|
proc=db_lock(5)
|
|
short=phase('db_burst_5s',45); proc.wait(timeout=10)
|
|
assert not fired(short,'APIAvailabilityBurn'),'Short spike generated false alert'
|
|
phase('settle',35)
|
|
proc=db_lock(70)
|
|
sustained=phase('db_sustained_70s',75); proc.wait(timeout=10)
|
|
assert fired(sustained,'APIAvailabilityBurn'),'Sustained DB failure not detected'
|
|
recovered=phase('db_recovery',70)
|
|
assert recovered['samples'][-1]['alerts'].get('APIAvailabilityBurn')!='firing'
|
|
errors=[r for r in requests if sustained['start']<=r['at']<sustained['end'] and r['status']==503]
|
|
assert errors,'No injected DB error was observed'
|
|
trace_id=errors[len(errors)//2]['trace_id']
|
|
response=httpx.get('http://127.0.0.1:3200/api/traces/'+trace_id,headers={'Accept':'application/json'},timeout=15)
|
|
response.raise_for_status(); trace=response.json()
|
|
db_spans=[s for s in trace_spans(trace) if s['name']=='db.session.authenticate']
|
|
assert db_spans and any(s.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR') for s in db_spans)
|
|
(OUT/'db-error-trace.json').write_text(json.dumps(trace,indent=2)+'\n',encoding='utf-8')
|
|
docker('compose','stop','worker')
|
|
try:
|
|
aids=[upload(c,headers,chat).json()['id'] for _ in range(8)]
|
|
queued=phase('worker_stopped',85)
|
|
assert fired(queued,'QueueLag') and fired(queued,'AttachmentReadyBurn')
|
|
finally: docker('compose','start','worker')
|
|
started=time.time()
|
|
for aid in aids: wait_ready(c,headers,aid)
|
|
results['queue_seconds_to_drain']=time.time()-started
|
|
queue_recovered=phase('worker_recovery',110)
|
|
assert not {'QueueLag','AttachmentReadyBurn'} & {n for n,s in queue_recovered['samples'][-1]['alerts'].items() if s=='firing'}
|
|
docker('compose','stop','-t','3','loki','tempo')
|
|
try:
|
|
backend=phase('telemetry_backends_stopped',80)
|
|
assert backend['requests']['success']>0 and backend['requests']['errors']==0
|
|
finally: docker('compose','start','loki','tempo')
|
|
phase('telemetry_recovery',45)
|
|
lines=[line for line in docker('compose','logs','--no-color','otel-collector').splitlines()
|
|
if 'Exporting failed. Dropping data.' in line or 'sending_queue is full' in line]
|
|
(OUT/'collector-drops.txt').write_text('\n'.join(lines)+'\n',encoding='utf-8')
|
|
dropped={}
|
|
for line in lines:
|
|
if '"dropped_items"' in line:
|
|
entry=json.loads(line[line.index('{'):]); signal=entry['otelcol.signal']
|
|
dropped[signal]=dropped.get(signal,0)+entry['dropped_items']
|
|
results['collector_explicit_retry_drops']=dropped
|
|
for sample in backend['samples']:
|
|
caps={row['metric'].get('exporter'):float(row['value'][1]) for row in sample['collector_capacity']}
|
|
for row in sample['collector_queue']:
|
|
assert float(row['value'][1])<=caps[row['metric'].get('exporter')]
|
|
results['summary']={'short_burst_false_firings':len(fired(short,'APIAvailabilityBurn')),
|
|
'api_detection_seconds':fired(sustained,'APIAvailabilityBurn')[0],
|
|
'queue_lag_detection_seconds':fired(queued,'QueueLag')[0],
|
|
'ready_slo_detection_seconds':fired(queued,'AttachmentReadyBurn')[0],
|
|
'db_error_trace_id':trace_id,'backend_outage_business_requests':backend['requests'],'passed':True}
|
|
results['passed']=True
|
|
finally:
|
|
stop.set()
|
|
for f in futures: f.result(timeout=15)
|
|
(OUT/'timeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
|
|
(OUT/'requests.json').write_text(json.dumps(requests,indent=2)+'\n',encoding='utf-8')
|
|
# After traffic stops, allow SDK/batch/scrape to flush. There are no health
|
|
# spans. Compare record counters after exporter queues have drained.
|
|
time.sleep(15)
|
|
def total(expr): return sum(float(r['value'][1]) for r in query(expr))
|
|
assert total('otelcol_exporter_queue_size')==0
|
|
results['quiescent_collector']={}
|
|
for signal in ('spans','log_records'):
|
|
accepted=total('otelcol_receiver_accepted_'+signal+'_total')
|
|
sent=total('otelcol_exporter_sent_'+signal+'_total')
|
|
results['quiescent_collector'][signal]={'accepted':accepted,'sent':sent,'not_exported':accepted-sent}
|
|
(OUT/'timeline.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(results['summary'],indent=2),flush=True)
|
|
|
|
if __name__=='__main__': main()
|