Files
mtusi2026lab6/student-tests/observability.py
T

161 lines
13 KiB
Python

#!/usr/bin/env python3
"""Saved telemetry evidence and bounded incidents, all using synthetic traffic."""
import concurrent.futures,datetime,json,os,subprocess,sys,threading,time,urllib.parse,urllib.request,uuid
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'tests'))
from client import Client,Failure,check
BASE=os.environ.get('BASE_URL','http://localhost:8086');PROM=os.environ.get('PROMETHEUS_URL','http://localhost:9096');TEMPO=os.environ.get('TEMPO_URL','http://localhost:3206');LOKI=os.environ.get('LOKI_URL','http://localhost:3106')
RESULT={'started_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),'slo':{'window':'10m rolling (local exercise)','api':'99% non-5xx of business requests','ready':'95% job versions ready within 5s; cohorts mature at 5s, unfinished count bad'},'incidents':[]}
def get(url):
with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as r:return json.load(r)
def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip()
def query(expr):return get(PROM+'/api/v1/query?'+urllib.parse.urlencode({'query':expr}))['data']['result']
def scalar(expr):
values=query(expr);return float(values[0]['value'][1]) if values else None
def alerts():return [{'name':x['labels']['alertname'],'state':x['state'],'active_at':x.get('activeAt')} for x in get(PROM+'/api/v1/alerts')['data']['alerts']]
def wait_for(fn,timeout=60):
start=time.monotonic()
while time.monotonic()-start<timeout:
try:
value=fn()
if value:return value,round(time.monotonic()-start,3)
except (OSError,KeyError,ValueError):pass
time.sleep(1)
raise AssertionError('timeout waiting telemetry')
def norm_spans(trace):
spans=[]
for b in trace.get('batches',trace.get('resourceSpans',[])):
attrs={x['key']:x['value'].get('stringValue') for x in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
for s in scope.get('spans',[]):
spans.append({'service':attrs.get('service.name'),'name':s['name'],'id':s.get('spanId'),'parent_id':s.get('parentSpanId'),'kind':s.get('kind'),'status':s.get('status',{}),'duration_ms':(int(s.get('endTimeUnixNano',0))-int(s.get('startTimeUnixNano',0)))/1e6})
return spans
def trace_chain(c,a,chat,retry=False,artifact="trace-chain.json"):
if retry:dc("stop","worker")
traceid=uuid.uuid4().hex;parent=uuid.uuid4().hex[:16];raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
headers={'Content-Type':'image/png'}
if retry:headers.update({'traceparent':'00-'+traceid+'-'+parent+'-01','tracestate':'lab=defense'})
res=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,actor=a,raw=raw,headers=headers,schema='Attachment')
if retry:check(res['headers'].get('X-Trace-Id')==traceid,'incoming W3C trace context lost')
else:traceid=res['headers'].get('X-Trace-Id');check(traceid and len(traceid)==32,'missing real root trace')
if retry:
dc('stop','s3');dc('up','-d','--no-deps','worker')
try:
def errored():
value=dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"SELECT count(*) FROM pipeline_audit WHERE job_id=(SELECT job_id FROM pipeline_jobs WHERE attachment_id='"+res['json']['id']+"') AND event='processing_error'")
return int(value)>0
wait_for(errored,20)
finally:dc('up','-d','--no-deps','--wait','s3')
wait_for(lambda:get(BASE+'/health/ready'),20)
att=c.wait_ready(a,res['json'])
def complete_trace():
t=get(TEMPO+'/api/traces/'+traceid)
serialized=json.dumps(t)
check(a['token'] not in serialized and a['registration']['password'] not in serialized,'credential leaked to traces')
spans=norm_spans(t)
names={s['name'] for s in spans};return spans if {'queue.publish','queue.consume','image.transform','s3.request'}.issubset(names) and (not retry or sum(x['name']=='queue.consume' for x in spans)>=2) else None
spans,delay=wait_for(complete_trace)
check({'messenger-api','messenger-worker','messenger-dispatcher'}.issubset({s['service'] for s in spans}),'missing service in trace')
for s in spans:
if s['name']=='queue.consume':check(s['parent_id'] in {x['id'] for x in spans if x['name']=='queue.publish'},'consumer not child of producer')
params={'query':'{service_name=~"messenger-(api|worker|dispatcher)"} | trace_id="'+traceid+'"','limit':'100','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))}
logs,_=wait_for(lambda:get(LOKI+'/loki/api/v1/query_range?'+urllib.parse.urlencode(params))['data']['result'])
serialized=json.dumps(logs)
check(a['token'] not in serialized and a['registration']['password'] not in serialized,'credential leaked to logs')
services={item['stream']['service_name'] for item in logs};check({'messenger-api','messenger-worker'}.issubset(services),'correlated service logs missing')
result={'trace_id':traceid,'attachment_id':att['id'],'ingest_wait_seconds':delay,'spans':spans,'log_services':sorted(services),'log_lines':sum(len(x['values']) for x in logs)}
(ROOT/'evidence'/artifact).write_text(json.dumps(result,indent=2)+'\n');print('PASS: real W3C API → outbox → producer → consumer → S3/DB trace and correlated logs',flush=True)
return result
class Traffic:
def __init__(self,actor,chat,threads=4):self.actor=actor;self.chat=chat;self.stop=threading.Event();self.results=[];self.pool=concurrent.futures.ThreadPoolExecutor(threads);self.tasks=[self.pool.submit(self.run) for _ in range(threads)]
def run(self):
c=Client(BASE,timeout=4)
while not self.stop.is_set():
start=time.monotonic()
try:r=c.message(self.actor,self.chat,'synthetic incident probe');self.results.append({'ok':True,'seconds':time.monotonic()-start,'trace_id':r['headers'].get('X-Trace-Id')})
except Failure:self.results.append({'ok':False,'seconds':time.monotonic()-start})
time.sleep(.2)
def close(self):self.stop.set();self.pool.shutdown(wait=True)
def timeline(seconds,step=1):
start=time.monotonic();points=[]
while time.monotonic()-start<seconds:
points.append({'elapsed':round(time.monotonic()-start,3),'time':time.time(),'alerts':alerts(),'queue_depth':scalar('max(messenger_queue_depth)'),'oldest_seconds':scalar('max(messenger_queue_oldest_age_seconds)'),'api_error_ratio':scalar('sum(rate(haproxy_backend_http_responses_total{proxy="replicas",code="5xx"}[10s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="replicas"}[10s])),.001)')});time.sleep(step)
return points
def worker_incident(c,a,chat):
timeline(15)
dc('stop','worker');start=time.monotonic();points=[];accepted=[]
try:
for _ in range(32):
att,data=c.attachment(a,chat);accepted.append((att,data));points+=timeline(1)
firing=[p for p in points if any(x['name']=='QueueLag' and x['state']=='firing' for x in p['alerts'])];check(firing,'QueueLag never fired')
burn=[p for p in points if any(x['name']=='ReadyBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])];check(burn,'ready burn alert never fired')
finally:dc('up','-d','--no-deps','worker')
restored=time.monotonic()
for att,data in accepted:
ready=c.wait_ready(a,att)
response=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a);check(response['bytes']==data,'accepted job lost')
recovery=timeline(22);check(not any(x['state']=='firing' and x['name'] in ['QueueLag','ReadyBudgetFastBurn'] for x in alerts()),'queue alerts did not resolve')
result={'name':'worker_stopped','accepted_jobs':len(accepted),'all_ready_and_identical':True,'detection_seconds':firing[0]['time']-(time.time()-(time.monotonic()-start)),'burn_detection_seconds':burn[0]['time']-(time.time()-(time.monotonic()-start)),'recovery_observation_seconds':round(time.monotonic()-restored,3),'timeline':points+recovery};RESULT['incidents'].append(result);print('PASS: stopped worker, lag/ready budget firing then resolved, all accepted originals preserved',flush=True)
def lock_db(seconds):
return subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f'BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); ROLLBACK;'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
def db_incident(c,a,chat):
traffic=Traffic(a,chat);timeline(20)
try:
# A 3s bounded spike should not page the multi-window burn alert.
spike_begin=len(traffic.results);proc=lock_db(3);spike=timeline(18);spike_results=traffic.results[spike_begin:];proc.wait(timeout=8)
false=[p for p in spike if any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])]
# Collect spans of an actual 5xx while the 2s SQL timeout fires.
proc=lock_db(45);start=time.monotonic();sustained=[];traceid=None
deadline=time.monotonic()+44
while time.monotonic()<deadline:
r=c.call('POST','/api/v1/chats/'+chat['id']+'/messages',[201,503],actor=a,body={'text':'DB lock probe'},headers={'Idempotency-Key':uuid.uuid4().hex})
if r['status']==503:traceid=r['headers'].get('X-Trace-Id')
sustained+=timeline(1)
proc.wait(timeout=8);restored=time.monotonic();recovery=timeline(32)
check(any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for p in sustained for x in p['alerts']),'sustained API burn never fired')
check(any(x['name']=='APIBudgetSlowBurn' and x['state']=='firing' for p in sustained for x in p['alerts']),'slow API burn never fired')
check(any(x['name']=='APIHighErrorRate' and x['state']=='firing' for p in sustained for x in p['alerts']),'high-error rule never fired')
check(not any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in alerts()),'API burn did not resolve')
trace,_=wait_for(lambda:norm_spans(get(TEMPO+'/api/traces/'+traceid)))
db=[s for s in trace if s['name'].startswith('db.')];check(any(s['duration_ms']>1000 for s in db),'DB trace does not show blocked query')
firing=[p for p in sustained if any(x['name']=='APIBudgetFastBurn' and x['state']=='firing' for x in p['alerts'])]
RESULT['incidents'].append({'name':'postgres_chat_table_lock','short_spike_seconds':3,'spike_http_errors':sum(not r['ok'] for r in spike_results),'spike_requests':len(spike_results),'false_fast_burn_firings':len(false),'sustained_lock_seconds':45,'detection_seconds':round(firing[0]['time']-(time.time()-(time.monotonic()-start)),3),'trace_id':traceid,'slow_db_spans':db,'timeline':spike+sustained+recovery})
print('PASS: DB lock distinguished by real spans, multi-window alert firing/resolved; spike false firings='+str(len(false)),flush=True)
finally:traffic.close()
def backend_incident(c,a,chat):
before=scalar('sum(otelcol_exporter_send_failed_spans_total)') or 0;traffic=Traffic(a,chat,threads=6);samples=[];dc('stop','tempo','loki')
try:
for _ in range(22):
samples.append(dc('stats','--no-stream','--format','{{.Name}} {{.MemUsage}}','collector'));time.sleep(1)
c.message(a,chat,'confirmed while telemetry backends unavailable')
finally:dc('up','-d','--no-deps','tempo','loki');traffic.close()
time.sleep(5);after=scalar('sum(otelcol_exporter_send_failed_spans_total)') or 0
check(all(r['ok'] for r in traffic.results),'telemetry outage broke business requests');check(after>before,'telemetry loss counter did not increase')
RESULT['incidents'].append({'name':'telemetry_backends_stopped','successful_messages':len(traffic.results),'business_errors':sum(not r['ok'] for r in traffic.results),'failed_span_exports_delta':after-before,'collector_memory_samples':samples,'queue_bound_batches':64,'export_retry_budget_seconds':15})
print('PASS: telemetry backend outage, business messages continue; bounded buffering and counted loss',flush=True)
def labels_check():
series=get(PROM+'/api/v1/series?'+urllib.parse.urlencode({'match[]':'{__name__=~"http_server.*|messenger.*"}'}))['data']
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','job_id','trace_id','span_id','filename','raw_url','url'}
check(series,'no application metric series')
for item in series:check(not forbidden.intersection(item),'high cardinality labels')
(ROOT/'evidence/metric-labels.json').write_text(json.dumps({'series_count':len(series),'label_keys':sorted(set().union(*(set(s) for s in series))),'forbidden_labels_absent':True},indent=2)+'\n')
print('PASS: live application metric series use bounded labels',flush=True)
def main():
global RESULT
phase=sys.argv[1] if len(sys.argv)>1 else 'all'
if phase!='all' and (ROOT/'evidence/incidents.json').exists():RESULT=json.loads((ROOT/'evidence/incidents.json').read_text())
c=Client(BASE);a,b,e,chat=c.bootstrap();wait_for(lambda:scalar('sum(http_server_requests_total)'))
if phase in ['all','trace']:
trace_chain(c,a,chat)
trace_chain(c,a,chat,retry=True,artifact='trace-chain-retry.json')
if phase in ['all','worker']:worker_incident(c,a,chat)
if phase in ['all','db']:db_incident(c,a,chat)
if phase in ['all','backend']:
backend_incident(c,a,chat)
trace_chain(c,a,chat,artifact='trace-chain-recovered.json')
labels_check()
RESULT['finished_at']=datetime.datetime.now(datetime.timezone.utc).isoformat()
if __name__=='__main__':
try:main()
finally:(ROOT/'evidence/incidents.json').write_text(json.dumps(RESULT,indent=2)+'\n')