46 lines
2.7 KiB
Python
46 lines
2.7 KiB
Python
"""Publication artifacts from saved observations only, never synthetic result values."""
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
|
|
root=Path(__file__).resolve().parents[1]
|
|
data=json.loads((root/'evidence/incidents/timeline.json').read_text(encoding='utf-8'))
|
|
phases=data['phases']; origin=phases[0]['start']
|
|
fig,axes=plt.subplots(4,1,figsize=(11,10),sharex=True,layout='constrained')
|
|
seen=set()
|
|
def metric(sample,key,aggregate=max):
|
|
values=[float(row['value'][1]) for row in sample.get(key,[]) if math.isfinite(float(row['value'][1]))]
|
|
return aggregate(values) if values else float('nan')
|
|
for phase in phases:
|
|
samples=phase['samples'];x=[(s['at']-origin)/60 for s in samples]
|
|
axes[0].plot(x,[metric(s,'api_error_ratio')*100 for s in samples],color='#C13E44')
|
|
axes[1].plot(x,[metric(s,'queue_age') for s in samples],color='#9163CB')
|
|
axes[2].plot(x,[metric(s,'collector_queue') for s in samples],color='#287AB1')
|
|
axes[3].plot(x,[metric(s,'collector_rss')/2**20 for s in samples],color='#15946A')
|
|
for ax in axes:
|
|
if phase['name'] in ('db_burst_5s','db_sustained_70s','worker_stopped','telemetry_backends_stopped'):
|
|
ax.axvspan((phase['start']-origin)/60,(phase['end']-origin)/60,color='#E1E6EC',alpha=.7,zorder=-1)
|
|
for name,idx in [('APIAvailabilityBurn',0),('QueueLag',1),('AttachmentReadyBurn',1)]:
|
|
xs=[(s['at']-origin)/60 for s in samples if s['alerts'].get(name)=='firing']
|
|
if xs:
|
|
axes[idx].scatter(xs,[104 if idx==0 else 115 if name=='QueueLag' else 125]*len(xs),s=12,marker='|',
|
|
color={'APIAvailabilityBurn':'#C13E44','QueueLag':'#287AB1','AttachmentReadyBurn':'#D38529'}[name],label=name if name not in seen else None)
|
|
seen.add(name)
|
|
axes[0].set_ylabel('API 5xx (%)');axes[1].set_ylabel('Oldest job (s)')
|
|
axes[2].set_ylabel('Collector queue\nmax batches');axes[3].set_ylabel('Collector RSS (MiB)')
|
|
axes[2].axhline(32,color='#C13E44',linestyle='--',label='Queue capacity32')
|
|
axes[3].axhline(256,color='#C13E44',linestyle='--',label='Container limit256MiB')
|
|
axes[3].set_xlabel('Minutes since experiment start')
|
|
for ax in axes:
|
|
ax.grid(alpha=.2);ax.set_ylim(bottom=0)
|
|
handles,labels=ax.get_legend_handles_labels()
|
|
if handles: ax.legend(loc='upper left',fontsize=8)
|
|
fig.suptitle('Measured SLO incidents and bounded telemetry buffers\nShading: fault observation windows; markers: firing alerts',fontsize=13)
|
|
fig.savefig(root/'evidence/incidents/timeline.png',dpi=160)
|
|
fig.savefig(root/'evidence/incidents/timeline.svg')
|
|
svg=root/'evidence/incidents/timeline.svg'
|
|
svg.write_text('\n'.join(line.rstrip() for line in svg.read_text(encoding='utf-8').splitlines())+'\n',encoding='utf-8')
|