#!/usr/bin/env python3 """Save bounded-buffer and loss observations for the actual Tempo outage.""" import datetime as dt,json,re,subprocess,sys from pathlib import Path ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts')) from observations import get x=json.loads((ROOT/'evidence/slo-experiment.json').read_text()) if not x.get('completed'):raise SystemExit('Experiment has not completed successfully') p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain') metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity'] def at(name,t): rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result'] return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo'] before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics} def total(rows):return sum(float(r['value'][1]) for r in rows) deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')} samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result'] peak=max((float(v[1]) for r in samples for v in r['values']),default=0) raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True) drops=[] for line in raw.splitlines(): try:d=json.loads(line[line.index('{'):]) except (ValueError,KeyError):continue if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d): drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d}) mem=[] for record in x['resources']: if p['start']<=record['at']<=finish: for c in record['containers']: if 'otel-collector' in c['Name']: m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale) out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')} (ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n') print('Saved backend outage balance, queue, memory and business persistence evidence')