161 lines
13 KiB
Python
161 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Saved telemetry evidence and bounded incidents, all using synthetic traffic."""
|
|
import concurrent.futures,datetime,json,os,subprocess,sys,threading,time,urllib.parse,urllib.request,uuid
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,Failure,check
|
|
BASE=os.environ.get('BASE_URL','http://localhost:8086');PROM=os.environ.get('PROMETHEUS_URL','http://localhost:9096');TEMPO=os.environ.get('TEMPO_URL','http://localhost:3206');LOKI=os.environ.get('LOKI_URL','http://localhost:3106')
|
|
RESULT={'started_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),'slo':{'window':'10m rolling (local exercise)','api':'99% non-5xx of business requests','ready':'95% job versions ready within 5s; cohorts mature at 5s, unfinished count bad'},'incidents':[]}
|
|
def get(url):
|
|
with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as r:return json.load(r)
|
|
def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip()
|
|
def query(expr):return get(PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':expr}))['data']['result']
|
|
def scalar(expr):
|
|
values=query(expr);return float(values[0]['value'][1]) if values else None
|
|
def alerts():return [{'name':x['labels']['alertname'],'state':x['state'],'active_at':x.get('activeAt')} for x in get(PROM+'/api/v1/alerts')['data']['alerts']]
|
|
def wait_for(fn,timeout=60):
|
|
start=time.monotonic()
|
|
while time.monotonic()-start<timeout:
|
|
try:
|
|
value=fn()
|
|
if value:return value,round(time.monotonic()-start,3)
|
|
except (OSError,KeyError,ValueError):pass
|
|
time.sleep(1)
|
|
raise AssertionError('timeout waiting telemetry')
|
|
def norm_spans(trace):
|
|
spans=[]
|
|
for b in trace.get('batches',trace.get('resourceSpans',[])):
|
|
attrs={x['key']:x['value'].get('stringValue') for x in b.get('resource',{}).get('attributes',[])}
|
|
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
|
|
for s in scope.get('spans',[]):
|
|
spans.append({'service':attrs.get('service.name'),'name':s['name'],'id':s.get('spanId'),'parent_id':s.get('parentSpanId'),'kind':s.get('kind'),'status':s.get('status',{}),'duration_ms':(int(s.get('endTimeUnixNano',0))-int(s.get('startTimeUnixNano',0)))/1e6})
|
|
return spans
|
|
def trace_chain(c,a,chat,retry=False,artifact="trace-chain.json"):
|
|
if retry:dc("stop","worker")
|
|
traceid=uuid.uuid4().hex;parent=uuid.uuid4().hex[:16];raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
headers={'Content-Type':'image/png'}
|
|
if retry:headers.update({'traceparent':'00-'+traceid+'-'+parent+'-01','tracestate':'lab=defense'})
|
|
res=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,actor=a,raw=raw,headers=headers,schema='Attachment')
|
|
if retry:check(res['headers'].get('X-Trace-Id')==traceid,'incoming W3C trace context lost')
|
|
else:traceid=res['headers'].get('X-Trace-Id');check(traceid and len(traceid)==32,'missing real root trace')
|
|
if retry:
|
|
dc('stop','s3');dc('up','-d','--no-deps','worker')
|
|
try:
|
|
def errored():
|
|
value=dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"SELECT count(*) FROM pipeline_audit WHERE job_id=(SELECT job_id FROM pipeline_jobs WHERE attachment_id='"+res['json']['id']+"') AND event='processing_error'")
|
|
return int(value)>0
|
|
wait_for(errored,20)
|
|
finally:dc('up','-d','--no-deps','--wait','s3')
|
|
wait_for(lambda:get(BASE+'/health/ready'),20)
|
|
att=c.wait_ready(a,res['json'])
|
|
def complete_trace():
|
|
t=get(TEMPO+'/api/traces/'+traceid)
|
|
serialized=json.dumps(t)
|
|
check(a['token'] not in serialized and a['registration']['password'] not in serialized,'credential leaked to traces')
|
|
spans=norm_spans(t)
|
|
names={s['name'] for s in spans};return spans if {'queue.publish','queue.consume','image.transform','s3.request'}.issubset(names) and (not retry or sum(x['name']=='queue.consume' for x in spans)>=2) else None
|
|
spans,delay=wait_for(complete_trace)
|
|
check({'messenger-api','messenger-worker','messenger-dispatcher'}.issubset({s['service'] for s in spans}),'missing service in trace')
|
|
for s in spans:
|
|
if s['name']=='queue.consume':check(s['parent_id'] in {x['id'] for x in spans if x['name']=='queue.publish'},'consumer not child of producer')
|
|
params={'query':'{service_name=~"messenger-(api|worker|dispatcher)"} | trace_id="'+traceid+'"','limit':'100','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))}
|
|
logs,_=wait_for(lambda:get(LOKI+'/loki/api/v1/query_range?'+urllib.parse.urlencode(params))['data']['result'])
|
|
serialized=json.dumps(logs)
|
|
check(a['token'] not in serialized and a['registration']['password'] not in serialized,'credential leaked to logs')
|
|
services={item['stream']['service_name'] for item in logs};check({'messenger-api','messenger-worker'}.issubset(services),'correlated service logs missing')
|
|
result={'trace_id':traceid,'attachment_id':att['id'],'ingest_wait_seconds':delay,'spans':spans,'log_services':sorted(services),'log_lines':sum(len(x['values']) for x in logs)}
|
|
(ROOT/'evidence'/artifact).write_text(json.dumps(result,indent=2)+'\n');print('PASS: real W3C API → outbox → producer → consumer → S3/DB trace and correlated logs',flush=True)
|
|
return result
|
|
class Traffic:
|
|
def __init__(self,actor,chat,threads=4):self.actor=actor;self.chat=chat;self.stop=threading.Event();self.results=[];self.pool=concurrent.futures.ThreadPoolExecutor(threads);self.tasks=[self.pool.submit(self.run) for _ in range(threads)]
|
|
def run(self):
|
|
c=Client(BASE,timeout=4)
|
|
while not self.stop.is_set():
|
|
start=time.monotonic()
|
|
try:r=c.message(self.actor,self.chat,'synthetic incident probe');self.results.append({'ok':True,'seconds':time.monotonic()-start,'trace_id':r['headers'].get('X-Trace-Id')})
|
|
except Failure:self.results.append({'ok':False,'seconds':time.monotonic()-start})
|
|
time.sleep(.2)
|
|
def close(self):self.stop.set();self.pool.shutdown(wait=True)
|
|
def timeline(seconds,step=1):
|
|
start=time.monotonic();points=[]
|
|
while time.monotonic()-start<seconds:
|
|
points.append({'elapsed':round(time.monotonic()-start,3),'time':time.time(),'alerts':alerts(),'queue_depth':scalar('max(messenger_queue_depth)'),'oldest_seconds':scalar('max(messenger_queue_oldest_age_seconds)'),'api_error_ratio':scalar('sum(rate(haproxy_backend_http_responses_total{proxy="replicas",code="5xx"}[10s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="replicas"}[10s])),.001)')});time.sleep(step)
|
|
return points
|
|
def worker_incident(c,a,chat):
|
|
timeline(15)
|
|
dc('stop','worker');start=time.monotonic();points=[];accepted=[]
|
|
try:
|
|
for _ in range(32):
|
|
att,data=c.attachment(a,chat);accepted.append((att,data));points+=timeline(1)
|
|
firing=[p for p in points if any(x['name']=='QueueLag' and x['state']=='firing' for x in p['alerts'])];check(firing,'QueueLag never fired')
|
|
burn=[p for p in points if any(x['name']=='ReadyBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])];check(burn,'ready burn alert never fired')
|
|
finally:dc('up','-d','--no-deps','worker')
|
|
restored=time.monotonic()
|
|
for att,data in accepted:
|
|
ready=c.wait_ready(a,att)
|
|
response=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a);check(response['bytes']==data,'accepted job lost')
|
|
recovery=timeline(22);check(not any(x['state']=='firing' and x['name'] in ['QueueLag','ReadyBudgetFastBurn'] for x in alerts()),'queue alerts did not resolve')
|
|
result={'name':'worker_stopped','accepted_jobs':len(accepted),'all_ready_and_identical':True,'detection_seconds':firing[0]['time']-(time.time()-(time.monotonic()-start)),'burn_detection_seconds':burn[0]['time']-(time.time()-(time.monotonic()-start)),'recovery_observation_seconds':round(time.monotonic()-restored,3),'timeline':points+recovery};RESULT['incidents'].append(result);print('PASS: stopped worker, lag/ready budget firing then resolved, all accepted originals preserved',flush=True)
|
|
def lock_db(seconds):
|
|
return subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f'BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); ROLLBACK;'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
def db_incident(c,a,chat):
|
|
traffic=Traffic(a,chat);timeline(20)
|
|
try:
|
|
# A 3s bounded spike should not page the multi-window burn alert.
|
|
spike_begin=len(traffic.results);proc=lock_db(3);spike=timeline(18);spike_results=traffic.results[spike_begin:];proc.wait(timeout=8)
|
|
false=[p for p in spike if any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])]
|
|
# Collect spans of an actual 5xx while the 2s SQL timeout fires.
|
|
proc=lock_db(45);start=time.monotonic();sustained=[];traceid=None
|
|
deadline=time.monotonic()+44
|
|
while time.monotonic()<deadline:
|
|
r=c.call('POST','/api/v1/chats/'+chat['id']+'/messages',[201,503],actor=a,body={'text':'DB lock probe'},headers={'Idempotency-Key':uuid.uuid4().hex})
|
|
if r['status']==503:traceid=r['headers'].get('X-Trace-Id')
|
|
sustained+=timeline(1)
|
|
proc.wait(timeout=8);restored=time.monotonic();recovery=timeline(32)
|
|
check(any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for p in sustained for x in p['alerts']),'sustained API burn never fired')
|
|
check(any(x['name']=='APIBudgetSlowBurn' and x['state']=='firing' for p in sustained for x in p['alerts']),'slow API burn never fired')
|
|
check(any(x['name']=='APIHighErrorRate' and x['state']=='firing' for p in sustained for x in p['alerts']),'high-error rule never fired')
|
|
check(not any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in alerts()),'API burn did not resolve')
|
|
trace,_=wait_for(lambda:norm_spans(get(TEMPO+'/api/traces/'+traceid)))
|
|
db=[s for s in trace if s['name'].startswith('db.')];check(any(s['duration_ms']>1000 for s in db),'DB trace does not show blocked query')
|
|
firing=[p for p in sustained if any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])]
|
|
RESULT['incidents'].append({'name':'postgres_chat_table_lock','short_spike_seconds':3,'spike_http_errors':sum(not r['ok'] for r in spike_results),'spike_requests':len(spike_results),'false_fast_burn_firings':len(false),'sustained_lock_seconds':45,'detection_seconds':round(firing[0]['time']-(time.time()-(time.monotonic()-start)),3),'trace_id':traceid,'slow_db_spans':db,'timeline':spike+sustained+recovery})
|
|
print('PASS: DB lock distinguished by real spans, multi-window alert firing/resolved; spike false firings='+str(len(false)),flush=True)
|
|
finally:traffic.close()
|
|
def backend_incident(c,a,chat):
|
|
before=scalar('sum(otelcol_exporter_send_failed_spans_total)') or 0;traffic=Traffic(a,chat,threads=6);samples=[];dc('stop','tempo','loki')
|
|
try:
|
|
for _ in range(22):
|
|
samples.append(dc('stats','--no-stream','--format','{{.Name}} {{.MemUsage}}','collector'));time.sleep(1)
|
|
c.message(a,chat,'confirmed while telemetry backends unavailable')
|
|
finally:dc('up','-d','--no-deps','tempo','loki');traffic.close()
|
|
time.sleep(5);after=scalar('sum(otelcol_exporter_send_failed_spans_total)') or 0
|
|
check(all(r['ok'] for r in traffic.results),'telemetry outage broke business requests');check(after>before,'telemetry loss counter did not increase')
|
|
RESULT['incidents'].append({'name':'telemetry_backends_stopped','successful_messages':len(traffic.results),'business_errors':sum(not r['ok'] for r in traffic.results),'failed_span_exports_delta':after-before,'collector_memory_samples':samples,'queue_bound_batches':64,'export_retry_budget_seconds':15})
|
|
print('PASS: telemetry backend outage, business messages continue; bounded buffering and counted loss',flush=True)
|
|
def labels_check():
|
|
series=get(PROM+'/api/v1/series?'+urllib.parse.urlencode({'match[]':'{__name__=~"http_server.*|messenger.*"}'}))['data']
|
|
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','job_id','trace_id','span_id','filename','raw_url','url'}
|
|
check(series,'no application metric series')
|
|
for item in series:check(not forbidden.intersection(item),'high cardinality labels')
|
|
(ROOT/'evidence/metric-labels.json').write_text(json.dumps({'series_count':len(series),'label_keys':sorted(set().union(*(set(s) for s in series))),'forbidden_labels_absent':True},indent=2)+'\n')
|
|
print('PASS: live application metric series use bounded labels',flush=True)
|
|
def main():
|
|
global RESULT
|
|
phase=sys.argv[1] if len(sys.argv)>1 else 'all'
|
|
if phase!='all' and (ROOT/'evidence/incidents.json').exists():RESULT=json.loads((ROOT/'evidence/incidents.json').read_text())
|
|
c=Client(BASE);a,b,e,chat=c.bootstrap();wait_for(lambda:scalar('sum(http_server_requests_total)'))
|
|
if phase in ['all','trace']:
|
|
trace_chain(c,a,chat)
|
|
trace_chain(c,a,chat,retry=True,artifact='trace-chain-retry.json')
|
|
if phase in ['all','worker']:worker_incident(c,a,chat)
|
|
if phase in ['all','db']:db_incident(c,a,chat)
|
|
if phase in ['all','backend']:
|
|
backend_incident(c,a,chat)
|
|
trace_chain(c,a,chat,artifact='trace-chain-recovered.json')
|
|
labels_check()
|
|
RESULT['finished_at']=datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
if __name__=='__main__':
|
|
try:main()
|
|
finally:(ROOT/'evidence/incidents.json').write_text(json.dumps(RESULT,indent=2)+'\n')
|