#!/usr/bin/env python3 """Saved telemetry evidence and bounded incidents, all using synthetic traffic.""" import concurrent.futures,datetime,json,os,subprocess,sys,threading,time,urllib.parse,urllib.request,uuid from pathlib import Path ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'tests')) from client import Client,Failure,check BASE=os.environ.get('BASE_URL','http://localhost:8086');PROM=os.environ.get('PROMETHEUS_URL','http://localhost:9096');TEMPO=os.environ.get('TEMPO_URL','http://localhost:3206');LOKI=os.environ.get('LOKI_URL','http://localhost:3106') RESULT={'started_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),'slo':{'window':'10m rolling (local exercise)','api':'99% non-5xx of business requests','ready':'95% job versions ready within 5s; cohorts mature at 5s, unfinished count bad'},'incidents':[]} def get(url): with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as r:return json.load(r) def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip() def query(expr):return get(PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':expr}))['data']['result'] def scalar(expr): values=query(expr);return float(values[0]['value'][1]) if values else None def alerts():return [{'name':x['labels']['alertname'],'state':x['state'],'active_at':x.get('activeAt')} for x in get(PROM+'/api/v1/alerts')['data']['alerts']] def wait_for(fn,timeout=60): start=time.monotonic() while time.monotonic()-start0 wait_for(errored,20) finally:dc('up','-d','--no-deps','--wait','s3') wait_for(lambda:get(BASE+'/health/ready'),20) att=c.wait_ready(a,res['json']) def complete_trace(): t=get(TEMPO+'/api/traces/'+traceid) serialized=json.dumps(t) check(a['token'] not in serialized and a['registration']['password'] not in serialized,'credential leaked to traces') spans=norm_spans(t) names={s['name'] for s in spans};return spans if {'queue.publish','queue.consume','image.transform','s3.request'}.issubset(names) and (not retry or sum(x['name']=='queue.consume' for x in spans)>=2) else None spans,delay=wait_for(complete_trace) check({'messenger-api','messenger-worker','messenger-dispatcher'}.issubset({s['service'] for s in spans}),'missing service in trace') for s in spans: if s['name']=='queue.consume':check(s['parent_id'] in {x['id'] for x in spans if x['name']=='queue.publish'},'consumer not child of producer') params={'query':'{service_name=~"messenger-(api|worker|dispatcher)"} | trace_id="'+traceid+'"','limit':'100','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))} logs,_=wait_for(lambda:get(LOKI+'/loki/api/v1/query_range?'+urllib.parse.urlencode(params))['data']['result']) serialized=json.dumps(logs) check(a['token'] not in serialized and a['registration']['password'] not in serialized,'credential leaked to logs') services={item['stream']['service_name'] for item in logs};check({'messenger-api','messenger-worker'}.issubset(services),'correlated service logs missing') result={'trace_id':traceid,'attachment_id':att['id'],'ingest_wait_seconds':delay,'spans':spans,'log_services':sorted(services),'log_lines':sum(len(x['values']) for x in logs)} (ROOT/'evidence'/artifact).write_text(json.dumps(result,indent=2)+'\n');print('PASS: real W3C API → outbox → producer → consumer → S3/DB trace and correlated logs',flush=True) return result class Traffic: def __init__(self,actor,chat,threads=4):self.actor=actor;self.chat=chat;self.stop=threading.Event();self.results=[];self.pool=concurrent.futures.ThreadPoolExecutor(threads);self.tasks=[self.pool.submit(self.run) for _ in range(threads)] def run(self): c=Client(BASE,timeout=4) while not self.stop.is_set(): start=time.monotonic() try:r=c.message(self.actor,self.chat,'synthetic incident probe');self.results.append({'ok':True,'seconds':time.monotonic()-start,'trace_id':r['headers'].get('X-Trace-Id')}) except Failure:self.results.append({'ok':False,'seconds':time.monotonic()-start}) time.sleep(.2) def close(self):self.stop.set();self.pool.shutdown(wait=True) def timeline(seconds,step=1): start=time.monotonic();points=[] while time.monotonic()-start1000 for s in db),'DB trace does not show blocked query') firing=[p for p in sustained if any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])] RESULT['incidents'].append({'name':'postgres_chat_table_lock','short_spike_seconds':3,'spike_http_errors':sum(not r['ok'] for r in spike_results),'spike_requests':len(spike_results),'false_fast_burn_firings':len(false),'sustained_lock_seconds':45,'detection_seconds':round(firing[0]['time']-(time.time()-(time.monotonic()-start)),3),'trace_id':traceid,'slow_db_spans':db,'timeline':spike+sustained+recovery}) print('PASS: DB lock distinguished by real spans, multi-window alert firing/resolved; spike false firings='+str(len(false)),flush=True) finally:traffic.close() def backend_incident(c,a,chat): before=scalar('sum(otelcol_exporter_send_failed_spans_total)') or 0;traffic=Traffic(a,chat,threads=6);samples=[];dc('stop','tempo','loki') try: for _ in range(22): samples.append(dc('stats','--no-stream','--format','{{.Name}} {{.MemUsage}}','collector'));time.sleep(1) c.message(a,chat,'confirmed while telemetry backends unavailable') finally:dc('up','-d','--no-deps','tempo','loki');traffic.close() time.sleep(5);after=scalar('sum(otelcol_exporter_send_failed_spans_total)') or 0 check(all(r['ok'] for r in traffic.results),'telemetry outage broke business requests');check(after>before,'telemetry loss counter did not increase') RESULT['incidents'].append({'name':'telemetry_backends_stopped','successful_messages':len(traffic.results),'business_errors':sum(not r['ok'] for r in traffic.results),'failed_span_exports_delta':after-before,'collector_memory_samples':samples,'queue_bound_batches':64,'export_retry_budget_seconds':15}) print('PASS: telemetry backend outage, business messages continue; bounded buffering and counted loss',flush=True) def labels_check(): series=get(PROM+'/api/v1/series?'+urllib.parse.urlencode({'match[]':'{__name__=~"http_server.*|messenger.*"}'}))['data'] forbidden={'user_id','chat_id','message_id','attachment_id','request_id','job_id','trace_id','span_id','filename','raw_url','url'} check(series,'no application metric series') for item in series:check(not forbidden.intersection(item),'high cardinality labels') (ROOT/'evidence/metric-labels.json').write_text(json.dumps({'series_count':len(series),'label_keys':sorted(set().union(*(set(s) for s in series))),'forbidden_labels_absent':True},indent=2)+'\n') print('PASS: live application metric series use bounded labels',flush=True) def main(): global RESULT phase=sys.argv[1] if len(sys.argv)>1 else 'all' if phase!='all' and (ROOT/'evidence/incidents.json').exists():RESULT=json.loads((ROOT/'evidence/incidents.json').read_text()) c=Client(BASE);a,b,e,chat=c.bootstrap();wait_for(lambda:scalar('sum(http_server_requests_total)')) if phase in ['all','trace']: trace_chain(c,a,chat) trace_chain(c,a,chat,retry=True,artifact='trace-chain-retry.json') if phase in ['all','worker']:worker_incident(c,a,chat) if phase in ['all','db']:db_incident(c,a,chat) if phase in ['all','backend']: backend_incident(c,a,chat) trace_chain(c,a,chat,artifact='trace-chain-recovered.json') labels_check() RESULT['finished_at']=datetime.datetime.now(datetime.timezone.utc).isoformat() if __name__=='__main__': try:main() finally:(ROOT/'evidence/incidents.json').write_text(json.dumps(RESULT,indent=2)+'\n')