Files
mtusi2026lab6/scripts/observability_backend.py
2026-11-18 21:37:25 +03:00

80 lines
3.5 KiB
Python

#!/usr/bin/env python3
"""Measure actual trace retention and bounded buffering during a backend outage."""
import datetime
import json
import subprocess
import sys
import time
from observability_common import ROOT, compose, get_json, ready, save, scalar, start_services
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
def snapshot():
return {'failed_spans': scalar('sum(otelcol_exporter_send_failed_spans_total)'),
'refused_spans': scalar('sum(otelcol_receiver_refused_spans_total)'),
'queued_batches': scalar('otelcol_exporter_queue_size{exporter="otlphttp/tempo"}'),
'rss_bytes': scalar('otelcol_process_memory_rss_bytes')}
def main():
ready()
c = Client('http://127.0.0.1:18086')
actor = c.user('backend')
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Backend outage', 'member_ids': [actor['id']]}, schema='Chat')['json']
report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'before': snapshot()}
trace_ids, timeline = [], []
try:
compose('stop', 'tempo')
started = time.monotonic()
for index in range(35):
response = c.message(actor, chat, 'synthetic backend outage')
trace_ids.append(response['headers']['X-Trace-Id'])
if index % 5 == 0:
timeline.append({'seconds': round(time.monotonic()-started, 2), **snapshot()})
time.sleep(1)
report['during'] = timeline
report['successful_writes'] = len(trace_ids)
finally:
start_services('tempo')
for _ in range(24):
time.sleep(2)
if snapshot()['queued_batches'] == 0:
break
time.sleep(6)
report['after'] = snapshot()
retained = 0
trace_results = []
for trace_id in trace_ids:
found = False
try:
d = get_json('http://127.0.0.1:13200/api/traces/' + trace_id)
found = bool(d.get('batches') or d.get('resourceSpans'))
except Exception:
pass
retained += int(found)
trace_results.append({'trace_id': trace_id, 'found': found})
report['request_trace_results'] = trace_results
report['retained_request_traces'] = retained
report['missing_request_traces'] = len(trace_ids)-retained
report['exporter_failed_spans_delta'] = report['after']['failed_spans']-report['before']['failed_spans']
report['receiver_refused_spans_delta'] = report['after']['refused_spans']-report['before']['refused_spans']
info = json.loads(subprocess.check_output(['docker', 'inspect', '--format',
'{"oom":{{json .State.OOMKilled}},"memory_limit":{{.HostConfig.Memory}}}', 'mtusi2026lab6-collector-1'], text=True))
report['collector'] = info
assert len(trace_ids) == 35, 'business writes failed'
assert not info['oom'], 'Collector exceeded memory cgroup'
assert max(x['queued_batches'] for x in timeline) <= 256, 'unbounded export queue'
assert report['after']['queued_batches'] == 0, 'export queue did not drain'
assert max(x['rss_bytes'] for x in timeline) < info['memory_limit'], 'RSS exceeded cgroup'
report['status'] = 'PASS'
report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
save('evidence/lab6-backend-outage.json', report)
print(json.dumps({k: report[k] for k in ('status', 'successful_writes', 'retained_request_traces', 'missing_request_traces', 'exporter_failed_spans_delta')}))
if __name__ == '__main__':
main()