forked from mtusi2026/mtusi2026lab6
35 lines
3.3 KiB
Python
35 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Save bounded-buffer and loss observations for the actual Tempo outage."""
|
|
import datetime as dt,json,re,subprocess,sys
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
|
|
from observations import get
|
|
x=json.loads((ROOT/'evidence/slo-experiment.json').read_text())
|
|
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
|
|
p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain')
|
|
metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity']
|
|
def at(name,t):
|
|
rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result']
|
|
return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo']
|
|
before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics}
|
|
def total(rows):return sum(float(r['value'][1]) for r in rows)
|
|
deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')}
|
|
samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result']
|
|
peak=max((float(v[1]) for r in samples for v in r['values']),default=0)
|
|
raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True)
|
|
drops=[]
|
|
for line in raw.splitlines():
|
|
try:d=json.loads(line[line.index('{'):])
|
|
except (ValueError,KeyError):continue
|
|
if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d):
|
|
drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d})
|
|
mem=[]
|
|
for record in x['resources']:
|
|
if p['start']<=record['at']<=finish:
|
|
for c in record['containers']:
|
|
if 'otel-collector' in c['Name']:
|
|
m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale)
|
|
out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')}
|
|
(ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n')
|
|
print('Saved backend outage balance, queue, memory and business persistence evidence')
|