feat: instrument api, relay and worker with opentelemetry

This commit is contained in:
2026-11-20 12:55:53 +03:00
parent 9ed32e2d15
commit 307b658bfb
18 changed files with 1843 additions and 49 deletions
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
"""
import json
import os
import sys
import time
import urllib.parse
import urllib.request
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
def get(path, **params):
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
with urllib.request.urlopen(url, timeout=15) as response:
return json.loads(response.read())
def run(target):
uid = target['datasource']['uid']
now = int(time.time())
if uid == 'prometheus':
expr = target['expr'].replace('$__rate_interval', '1m')
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
series = body['data']['result']
last = [s['values'][-1][1] for s in series if s['values']]
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
for s, v in zip(series[:4], last[:4]))
if uid == 'loki':
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
return sum(len(s['values']) for s in body['data']['result']), ''
if uid == 'tempo':
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
return len(body.get('traces', [])), ''
return 0, 'unknown datasource'
def main():
grafana = get('/api/dashboards/uid/messenger-overview')
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
empty = 0
for panel in dashboard['panels']:
for target in panel.get('targets', []):
count, sample = run(target)
empty += count == 0
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
sys.exit(0 if empty == 0 else 1)
if __name__ == '__main__':
main()
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus.
Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30]
"""
import json
import os
import subprocess
import sys
import tempfile
import time
import urllib.parse
import urllib.request
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}'
def prom(expr, at):
params = urllib.parse.urlencode({'query': expr, 'time': at})
with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r:
result = json.loads(r.read())['data']['result']
return float(result[0]['value'][1]) if result else None
def main():
duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30'
output = os.path.join(tempfile.mkdtemp(), 'load.json')
started = time.time()
subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output],
check=True, stdout=subprocess.DEVNULL)
report = json.load(open(output))
print('waiting 25 s for the metric export interval')
time.sleep(25)
at = time.time()
window = int(at - started) + 5
print('client (tests/load.py): %d requests, %.1f rps, errors %d' % (
report['successful_requests'], report['successful_rps'], report['failed_requests']))
count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at)
print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window))
print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms'))
for p in ('p50', 'p95', 'p99'):
q = int(p[1:]) / 100
server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at)
print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000))
print('server-side histogram excludes network/client overhead; values are interpolated inside buckets '
'(…, 5, 10, 25, 50, 100 ms, …).')
if __name__ == '__main__':
main()
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
"""
import base64
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
def call(method, url, body=None, headers=None, raw=None):
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
all_headers = dict(headers or {})
if body is not None:
all_headers['Content-Type'] = 'application/json'
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status, json.loads(response.read() or b'null')
except urllib.error.HTTPError as error:
return error.code, None
def session():
username = 'trace_' + uuid.uuid4().hex[:8]
password = 'Trace-check-' + uuid.uuid4().hex
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
credential = base64.b64encode((username + ':' + password).encode()).decode()
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
auth = {'Authorization': 'Bearer ' + login['token']}
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
return auth, chat['id']
def wait_trace(trace_id, services, timeout=60):
deadline = time.monotonic() + timeout
spans = []
while time.monotonic() < deadline:
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
if code == 200 and body:
spans = []
for batch in body.get('batches', []):
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
service = attrs['service.name']['stringValue']
for scope in batch.get('scopeSpans', []):
for span in scope.get('spans', []):
spans.append((service, span))
if services <= {service for service, _ in spans}:
return spans
time.sleep(3)
return spans
def b64hex(value):
return base64.b64decode(value).hex() if value else ''
def main():
trace_id = os.urandom(16).hex()
client_span = os.urandom(8).hex()
traceparent = '00-%s-%s-01' % (trace_id, client_span)
auth, chat = session()
started = time.monotonic()
code, attachment = call(
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
)
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
status = None
while time.monotonic() - started < 60:
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
status = body['status']
if status in ('ready', 'failed'):
break
time.sleep(0.5)
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
continue
parent = b64hex(span.get('parentSpanId'))
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
streams = []
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
streams = (logs or {}).get('data', {}).get('result', [])
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
break
time.sleep(3)
print('\nLoki %s' % query)
lines = 0
for stream in streams:
labels = stream['stream']
for _, line in stream['values']:
lines += 1
print(' %-17s %-8s %-13s span_id=%s %s' % (
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
labels.get('span_id', ''), line[:60]))
kinds = {(service, span.get('kind')) for service, span in spans}
ok = (
status == 'ready'
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
)
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
sys.exit(0 if ok else 1)
if __name__ == '__main__':
main()