33 lines
1.5 KiB
Python
33 lines
1.5 KiB
Python
#!/usr/bin/env python3
|
|
import re
|
|
import time
|
|
from client import Client, arguments, check, run
|
|
from obs import loki, prom, trace, wait_loki
|
|
from stack import compose, upload, wait_status
|
|
|
|
|
|
def main():
|
|
args = arguments('Telemetry history survives a restart of the storages')
|
|
c = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
request_id = upload(c, alice, chat)['headers']['X-Request-Id']
|
|
rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id)
|
|
trace_id = rows[-1][1]['trace_id']
|
|
check(re.fullmatch(r'[0-9a-f]{32}', trace_id), 'no trace_id in log')
|
|
time.sleep(15)
|
|
before = len(trace(trace_id))
|
|
samples = prom('count_over_time(http_server_requests_total[10m])')
|
|
check(before > 1 and samples, 'nothing to compare')
|
|
compose('restart', 'prometheus', 'loki', 'tempo')
|
|
compose('up', '-d', '--wait', 'prometheus', 'loki', 'tempo', 'grafana')
|
|
check(len(trace(trace_id, 90)) == before, 'trace lost after restart')
|
|
check(wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id, 90), 'log lost after restart')
|
|
check(prom('count_over_time(http_server_requests_total[10m])'), 'metrics lost after restart')
|
|
print('persistence: trace %s (%d spans), its request log and %d metric series survived a restart of prometheus, loki and tempo'
|
|
% (trace_id, before, len(samples)))
|
|
print('PASS: telemetry history is kept on volumes')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|