forked from mtusi2026/mtusi2026lab6
28 lines
2.2 KiB
Python
28 lines
2.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Derive incident timing from recorded observations, without inventing missing data."""
|
|
import json
|
|
from pathlib import Path
|
|
root=Path(__file__).resolve().parents[1]
|
|
x=json.loads((root/'evidence/slo-experiment.json').read_text())
|
|
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
|
|
rows=x['observations'];events=x['events'];results=[]
|
|
for i,event in enumerate(events):
|
|
name=event['phase']
|
|
if not any(name.startswith(s) for s in ['worker_short_','db_short_','worker_long_','db_long_']):continue
|
|
if 'recovery' in name:continue
|
|
prefix=name.rsplit('_',1)[0];start=event['at']
|
|
phase=next(p for p in x['phases'] if p['name']==name);recovered=phase['end']
|
|
nextfault=next((e['at'] for e in events[i+1:] if ('_short_' in e['phase'] or '_long_' in e['phase']) and 'recovery' not in e['phase']),x.get('finished',float('inf')))
|
|
observed=[r for r in rows if start<=r['at']<nextfault]
|
|
expected=['QueueLag','ReadyBurnRateFast','ReadyBurnRateSlow'] if name.startswith('worker') else ['ApiBurnRateFast','ApiBurnRateSlow']
|
|
alerts={}
|
|
for alert in expected:
|
|
def firing(r):return any(a['labels']['alertname']==alert and a['state']=='firing' for a in r.get('alerts',[]))
|
|
first=next((r['at'] for r in observed if firing(r)),None)
|
|
resolved=next((r['at'] for r in observed if first is not None and r['at']>max(first,recovered) and not firing(r)),None)
|
|
alerts[alert]={'detection_seconds':None if first is None else first-start,'resolved_after_restore_seconds':None if resolved is None else resolved-recovered,'fired':first is not None}
|
|
results.append({'phase':name,'duration':recovered-start,'alerts':alerts,'successful_rps':phase['successful_rps'],'error_ratio':phase['error_ratio'],'latency_success_ms':phase['latency_success_ms']})
|
|
summary={'completed':x.get('completed',False),'duration_seconds':x.get('finished',0)-x['started'],'parameters':x['parameters'],'incidents':results,'traffic':x.get('traffic'),'attachment_states':x.get('attachment_states'),'image_id':x.get('image_id')}
|
|
(root/'evidence/slo-summary.json').write_text(json.dumps(summary,indent=2)+'\n')
|
|
print(json.dumps(summary,indent=2))
|