#!/usr/bin/env python3 """Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki. Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py """ import base64 import json import os import sys import time import urllib.error import urllib.parse import urllib.request import uuid BASE = os.environ.get('BASE_URL', 'http://localhost:8080') GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read() def call(method, url, body=None, headers=None, raw=None): data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) all_headers = dict(headers or {}) if body is not None: all_headers['Content-Type'] = 'application/json' request = urllib.request.Request(url, data=data, method=method, headers=all_headers) try: with urllib.request.urlopen(request, timeout=15) as response: return response.status, json.loads(response.read() or b'null') except urllib.error.HTTPError as error: return error.code, None def session(): username = 'trace_' + uuid.uuid4().hex[:8] password = 'Trace-check-' + uuid.uuid4().hex _, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password}) credential = base64.b64encode((username + ':' + password).encode()).decode() _, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential}) auth = {'Authorization': 'Bearer ' + login['token']} _, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth) return auth, chat['id'] def wait_trace(trace_id, services, timeout=60): deadline = time.monotonic() + timeout spans = [] while time.monotonic() < deadline: code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id) if code == 200 and body: spans = [] for batch in body.get('batches', []): attrs = {a['key']: a['value'] for a in batch['resource']['attributes']} service = attrs['service.name']['stringValue'] for scope in batch.get('scopeSpans', []): for span in scope.get('spans', []): spans.append((service, span)) if services <= {service for service, _ in spans}: return spans time.sleep(3) return spans def b64hex(value): return base64.b64decode(value).hex() if value else '' def main(): trace_id = os.urandom(16).hex() client_span = os.urandom(8).hex() traceparent = '00-%s-%s-01' % (trace_id, client_span) auth, chat = session() started = time.monotonic() code, attachment = call( 'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat, headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE, ) print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent) status = None while time.monotonic() - started < 60: _, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth) status = body['status'] if status in ('ready', 'failed'): break time.sleep(0.5) print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started)) spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'}) by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans} print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans))) for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])): if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'): continue parent = b64hex(span.get('parentSpanId')) parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name'] duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6 print(' %-17s %-22s %-48s %8.2f ms parent=%s' % ( service, span.get('kind', ''), span['name'][:48], duration, parent_name)) query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'}) streams = [] deadline = time.monotonic() + 30 while time.monotonic() < deadline: _, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params) streams = (logs or {}).get('data', {}).get('result', []) if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}: break time.sleep(3) print('\nLoki %s' % query) lines = 0 for stream in streams: labels = stream['stream'] for _, line in stream['values']: lines += 1 print(' %-17s %-8s %-13s span_id=%s %s' % ( labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''), labels.get('span_id', ''), line[:60])) kinds = {(service, span.get('kind')) for service, span in spans} ok = ( status == 'ready' and ('messenger-api', 'SPAN_KIND_SERVER') in kinds and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans) and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams} ) print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL')) sys.exit(0 if ok else 1) if __name__ == '__main__': main()