Files
mtusi2026lab6/scripts/analyze-backend.py
T
2026-11-17 13:08:13 +03:00

35 lines
3.3 KiB
Python

#!/usr/bin/env python3
"""Save bounded-buffer and loss observations for the actual Tempo outage."""
import datetime as dt,json,re,subprocess,sys
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
from observations import get
x=json.loads((ROOT/'evidence/slo-experiment.json').read_text())
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain')
metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity']
def at(name,t):
rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result']
return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo']
before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics}
def total(rows):return sum(float(r['value'][1]) for r in rows)
deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')}
samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result']
peak=max((float(v[1]) for r in samples for v in r['values']),default=0)
raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True)
drops=[]
for line in raw.splitlines():
try:d=json.loads(line[line.index('{'):])
except (ValueError,KeyError):continue
if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d):
drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d})
mem=[]
for record in x['resources']:
if p['start']<=record['at']<=finish:
for c in record['containers']:
if 'otel-collector' in c['Name']:
m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale)
out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')}
(ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n')
print('Saved backend outage balance, queue, memory and business persistence evidence')