forked from mtusi2026/mtusi2026lab6
feat: instrument api, relay and worker with opentelemetry
This commit is contained in:
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
|
||||
|
||||
|
||||
def get(path, **params):
|
||||
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def run(target):
|
||||
uid = target['datasource']['uid']
|
||||
now = int(time.time())
|
||||
if uid == 'prometheus':
|
||||
expr = target['expr'].replace('$__rate_interval', '1m')
|
||||
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
|
||||
series = body['data']['result']
|
||||
last = [s['values'][-1][1] for s in series if s['values']]
|
||||
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
|
||||
for s, v in zip(series[:4], last[:4]))
|
||||
if uid == 'loki':
|
||||
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
|
||||
return sum(len(s['values']) for s in body['data']['result']), ''
|
||||
if uid == 'tempo':
|
||||
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
|
||||
return len(body.get('traces', [])), ''
|
||||
return 0, 'unknown datasource'
|
||||
|
||||
|
||||
def main():
|
||||
grafana = get('/api/dashboards/uid/messenger-overview')
|
||||
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
|
||||
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
|
||||
empty = 0
|
||||
for panel in dashboard['panels']:
|
||||
for target in panel.get('targets', []):
|
||||
count, sample = run(target)
|
||||
empty += count == 0
|
||||
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
|
||||
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
|
||||
sys.exit(0 if empty == 0 else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30]
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}'
|
||||
|
||||
|
||||
def prom(expr, at):
|
||||
params = urllib.parse.urlencode({'query': expr, 'time': at})
|
||||
with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r:
|
||||
result = json.loads(r.read())['data']['result']
|
||||
return float(result[0]['value'][1]) if result else None
|
||||
|
||||
|
||||
def main():
|
||||
duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30'
|
||||
output = os.path.join(tempfile.mkdtemp(), 'load.json')
|
||||
started = time.time()
|
||||
subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
report = json.load(open(output))
|
||||
print('waiting 25 s for the metric export interval')
|
||||
time.sleep(25)
|
||||
at = time.time()
|
||||
window = int(at - started) + 5
|
||||
print('client (tests/load.py): %d requests, %.1f rps, errors %d' % (
|
||||
report['successful_requests'], report['successful_rps'], report['failed_requests']))
|
||||
count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at)
|
||||
print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window))
|
||||
print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms'))
|
||||
for p in ('p50', 'p95', 'p99'):
|
||||
q = int(p[1:]) / 100
|
||||
server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at)
|
||||
print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000))
|
||||
print('server-side histogram excludes network/client overhead; values are interpolated inside buckets '
|
||||
'(…, 5, 10, 25, 50, 100 ms, …).')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
|
||||
|
||||
def call(method, url, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
all_headers = dict(headers or {})
|
||||
if body is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status, json.loads(response.read() or b'null')
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
|
||||
def session():
|
||||
username = 'trace_' + uuid.uuid4().hex[:8]
|
||||
password = 'Trace-check-' + uuid.uuid4().hex
|
||||
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
auth = {'Authorization': 'Bearer ' + login['token']}
|
||||
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
|
||||
return auth, chat['id']
|
||||
|
||||
|
||||
def wait_trace(trace_id, services, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
spans = []
|
||||
while time.monotonic() < deadline:
|
||||
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
||||
if code == 200 and body:
|
||||
spans = []
|
||||
for batch in body.get('batches', []):
|
||||
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
|
||||
service = attrs['service.name']['stringValue']
|
||||
for scope in batch.get('scopeSpans', []):
|
||||
for span in scope.get('spans', []):
|
||||
spans.append((service, span))
|
||||
if services <= {service for service, _ in spans}:
|
||||
return spans
|
||||
time.sleep(3)
|
||||
return spans
|
||||
|
||||
|
||||
def b64hex(value):
|
||||
return base64.b64decode(value).hex() if value else ''
|
||||
|
||||
|
||||
def main():
|
||||
trace_id = os.urandom(16).hex()
|
||||
client_span = os.urandom(8).hex()
|
||||
traceparent = '00-%s-%s-01' % (trace_id, client_span)
|
||||
auth, chat = session()
|
||||
started = time.monotonic()
|
||||
code, attachment = call(
|
||||
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
|
||||
)
|
||||
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
|
||||
status = None
|
||||
while time.monotonic() - started < 60:
|
||||
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
|
||||
status = body['status']
|
||||
if status in ('ready', 'failed'):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
|
||||
|
||||
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
|
||||
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
|
||||
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
|
||||
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
|
||||
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
|
||||
continue
|
||||
parent = b64hex(span.get('parentSpanId'))
|
||||
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
|
||||
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
||||
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
|
||||
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
|
||||
|
||||
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
|
||||
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
|
||||
streams = []
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
|
||||
streams = (logs or {}).get('data', {}).get('result', [])
|
||||
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
|
||||
break
|
||||
time.sleep(3)
|
||||
print('\nLoki %s' % query)
|
||||
lines = 0
|
||||
for stream in streams:
|
||||
labels = stream['stream']
|
||||
for _, line in stream['values']:
|
||||
lines += 1
|
||||
print(' %-17s %-8s %-13s span_id=%s %s' % (
|
||||
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
|
||||
labels.get('span_id', ''), line[:60]))
|
||||
|
||||
kinds = {(service, span.get('kind')) for service, span in spans}
|
||||
ok = (
|
||||
status == 'ready'
|
||||
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
|
||||
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
|
||||
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
|
||||
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
|
||||
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
|
||||
)
|
||||
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user