Files
mtusi2026lab6/tests/telemetry_persistence.py
T

33 lines
1.5 KiB
Python

#!/usr/bin/env python3
import re
import time
from client import Client, arguments, check, run
from obs import loki, prom, trace, wait_loki
from stack import compose, upload, wait_status
def main():
args = arguments('Telemetry history survives a restart of the storages')
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = c.bootstrap()
request_id = upload(c, alice, chat)['headers']['X-Request-Id']
rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id)
trace_id = rows[-1][1]['trace_id']
check(re.fullmatch(r'[0-9a-f]{32}', trace_id), 'no trace_id in log')
time.sleep(15)
before = len(trace(trace_id))
samples = prom('count_over_time(http_server_requests_total[10m])')
check(before > 1 and samples, 'nothing to compare')
compose('restart', 'prometheus', 'loki', 'tempo')
compose('up', '-d', '--wait', 'prometheus', 'loki', 'tempo', 'grafana')
check(len(trace(trace_id, 90)) == before, 'trace lost after restart')
check(wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id, 90), 'log lost after restart')
check(prom('count_over_time(http_server_requests_total[10m])'), 'metrics lost after restart')
print('persistence: trace %s (%d spans), its request log and %d metric series survived a restart of prometheus, loki and tempo'
% (trace_id, before, len(samples)))
print('PASS: telemetry history is kept on volumes')
if __name__ == '__main__':
run(main)