#!/usr/bin/env python3 """Measure actual trace retention and bounded buffering during a backend outage.""" import datetime import json import subprocess import sys import time from observability_common import ROOT, compose, get_json, ready, save, scalar, start_services sys.path.insert(0, str(ROOT / 'tests')) from client import Client def snapshot(): return {'failed_spans': scalar('sum(otelcol_exporter_send_failed_spans_total)'), 'refused_spans': scalar('sum(otelcol_receiver_refused_spans_total)'), 'queued_batches': scalar('otelcol_exporter_queue_size{exporter="otlphttp/tempo"}'), 'rss_bytes': scalar('otelcol_process_memory_rss_bytes')} def main(): ready() c = Client('http://127.0.0.1:18086') actor = c.user('backend') chat = c.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'Backend outage', 'member_ids': [actor['id']]}, schema='Chat')['json'] report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'before': snapshot()} trace_ids, timeline = [], [] try: compose('stop', 'tempo') started = time.monotonic() for index in range(35): response = c.message(actor, chat, 'synthetic backend outage') trace_ids.append(response['headers']['X-Trace-Id']) if index % 5 == 0: timeline.append({'seconds': round(time.monotonic()-started, 2), **snapshot()}) time.sleep(1) report['during'] = timeline report['successful_writes'] = len(trace_ids) finally: start_services('tempo') for _ in range(24): time.sleep(2) if snapshot()['queued_batches'] == 0: break time.sleep(6) report['after'] = snapshot() retained = 0 trace_results = [] for trace_id in trace_ids: found = False try: d = get_json('http://127.0.0.1:13200/api/traces/' + trace_id) found = bool(d.get('batches') or d.get('resourceSpans')) except Exception: pass retained += int(found) trace_results.append({'trace_id': trace_id, 'found': found}) report['request_trace_results'] = trace_results report['retained_request_traces'] = retained report['missing_request_traces'] = len(trace_ids)-retained report['exporter_failed_spans_delta'] = report['after']['failed_spans']-report['before']['failed_spans'] report['receiver_refused_spans_delta'] = report['after']['refused_spans']-report['before']['refused_spans'] info = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{"oom":{{json .State.OOMKilled}},"memory_limit":{{.HostConfig.Memory}}}', 'mtusi2026lab6-collector-1'], text=True)) report['collector'] = info assert len(trace_ids) == 35, 'business writes failed' assert not info['oom'], 'Collector exceeded memory cgroup' assert max(x['queued_batches'] for x in timeline) <= 256, 'unbounded export queue' assert report['after']['queued_batches'] == 0, 'export queue did not drain' assert max(x['rss_bytes'] for x in timeline) < info['memory_limit'], 'RSS exceeded cgroup' report['status'] = 'PASS' report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() save('evidence/lab6-backend-outage.json', report) print(json.dumps({k: report[k] for k in ('status', 'successful_writes', 'retained_request_traces', 'missing_request_traces', 'exporter_failed_spans_delta')})) if __name__ == '__main__': main()