forked from mtusi2026/mtusi2026lab6
80 lines
3.5 KiB
Python
80 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure actual trace retention and bounded buffering during a backend outage."""
|
|
import datetime
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from observability_common import ROOT, compose, get_json, ready, save, scalar, start_services
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
|
|
def snapshot():
|
|
return {'failed_spans': scalar('sum(otelcol_exporter_send_failed_spans_total)'),
|
|
'refused_spans': scalar('sum(otelcol_receiver_refused_spans_total)'),
|
|
'queued_batches': scalar('otelcol_exporter_queue_size{exporter="otlphttp/tempo"}'),
|
|
'rss_bytes': scalar('otelcol_process_memory_rss_bytes')}
|
|
|
|
|
|
def main():
|
|
ready()
|
|
c = Client('http://127.0.0.1:18086')
|
|
actor = c.user('backend')
|
|
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Backend outage', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'before': snapshot()}
|
|
trace_ids, timeline = [], []
|
|
try:
|
|
compose('stop', 'tempo')
|
|
started = time.monotonic()
|
|
for index in range(35):
|
|
response = c.message(actor, chat, 'synthetic backend outage')
|
|
trace_ids.append(response['headers']['X-Trace-Id'])
|
|
if index % 5 == 0:
|
|
timeline.append({'seconds': round(time.monotonic()-started, 2), **snapshot()})
|
|
time.sleep(1)
|
|
report['during'] = timeline
|
|
report['successful_writes'] = len(trace_ids)
|
|
finally:
|
|
start_services('tempo')
|
|
for _ in range(24):
|
|
time.sleep(2)
|
|
if snapshot()['queued_batches'] == 0:
|
|
break
|
|
time.sleep(6)
|
|
report['after'] = snapshot()
|
|
retained = 0
|
|
trace_results = []
|
|
for trace_id in trace_ids:
|
|
found = False
|
|
try:
|
|
d = get_json('http://127.0.0.1:13200/api/traces/' + trace_id)
|
|
found = bool(d.get('batches') or d.get('resourceSpans'))
|
|
except Exception:
|
|
pass
|
|
retained += int(found)
|
|
trace_results.append({'trace_id': trace_id, 'found': found})
|
|
report['request_trace_results'] = trace_results
|
|
report['retained_request_traces'] = retained
|
|
report['missing_request_traces'] = len(trace_ids)-retained
|
|
report['exporter_failed_spans_delta'] = report['after']['failed_spans']-report['before']['failed_spans']
|
|
report['receiver_refused_spans_delta'] = report['after']['refused_spans']-report['before']['refused_spans']
|
|
info = json.loads(subprocess.check_output(['docker', 'inspect', '--format',
|
|
'{"oom":{{json .State.OOMKilled}},"memory_limit":{{.HostConfig.Memory}}}', 'mtusi2026lab6-collector-1'], text=True))
|
|
report['collector'] = info
|
|
assert len(trace_ids) == 35, 'business writes failed'
|
|
assert not info['oom'], 'Collector exceeded memory cgroup'
|
|
assert max(x['queued_batches'] for x in timeline) <= 256, 'unbounded export queue'
|
|
assert report['after']['queued_batches'] == 0, 'export queue did not drain'
|
|
assert max(x['rss_bytes'] for x in timeline) < info['memory_limit'], 'RSS exceeded cgroup'
|
|
report['status'] = 'PASS'
|
|
report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
save('evidence/lab6-backend-outage.json', report)
|
|
print(json.dumps({k: report[k] for k in ('status', 'successful_writes', 'retained_request_traces', 'missing_request_traces', 'exporter_failed_spans_delta')}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|