#!/usr/bin/env python3 """Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus. Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30] """ import json import os import subprocess import sys import tempfile import time import urllib.parse import urllib.request GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') BASE = os.environ.get('BASE_URL', 'http://lb:8080') SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}' def prom(expr, at): params = urllib.parse.urlencode({'query': expr, 'time': at}) with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r: result = json.loads(r.read())['data']['result'] return float(result[0]['value'][1]) if result else None def main(): duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30' output = os.path.join(tempfile.mkdtemp(), 'load.json') started = time.time() subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output], check=True, stdout=subprocess.DEVNULL) report = json.load(open(output)) print('waiting 25 s for the metric export interval') time.sleep(25) at = time.time() window = int(at - started) + 5 print('client (tests/load.py): %d requests, %.1f rps, errors %d' % ( report['successful_requests'], report['successful_rps'], report['failed_requests'])) count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at) print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window)) print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms')) for p in ('p50', 'p95', 'p99'): q = int(p[1:]) / 100 server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at) print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000)) print('server-side histogram excludes network/client overhead; values are interpolated inside buckets ' '(…, 5, 10, 25, 50, 100 ms, …).') if __name__ == '__main__': main()