135 lines
6.0 KiB
Python
Executable File
135 lines
6.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
|
|
|
|
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
|
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
|
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
|
|
|
|
|
def call(method, url, body=None, headers=None, raw=None):
|
|
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
|
all_headers = dict(headers or {})
|
|
if body is not None:
|
|
all_headers['Content-Type'] = 'application/json'
|
|
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=15) as response:
|
|
return response.status, json.loads(response.read() or b'null')
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, None
|
|
|
|
|
|
def session():
|
|
username = 'trace_' + uuid.uuid4().hex[:8]
|
|
password = 'Trace-check-' + uuid.uuid4().hex
|
|
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
|
|
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
|
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
|
auth = {'Authorization': 'Bearer ' + login['token']}
|
|
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
|
|
return auth, chat['id']
|
|
|
|
|
|
def wait_trace(trace_id, services, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
spans = []
|
|
while time.monotonic() < deadline:
|
|
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
|
if code == 200 and body:
|
|
spans = []
|
|
for batch in body.get('batches', []):
|
|
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
|
|
service = attrs['service.name']['stringValue']
|
|
for scope in batch.get('scopeSpans', []):
|
|
for span in scope.get('spans', []):
|
|
spans.append((service, span))
|
|
if services <= {service for service, _ in spans}:
|
|
return spans
|
|
time.sleep(3)
|
|
return spans
|
|
|
|
|
|
def b64hex(value):
|
|
return base64.b64decode(value).hex() if value else ''
|
|
|
|
|
|
def main():
|
|
trace_id = os.urandom(16).hex()
|
|
client_span = os.urandom(8).hex()
|
|
traceparent = '00-%s-%s-01' % (trace_id, client_span)
|
|
auth, chat = session()
|
|
started = time.monotonic()
|
|
code, attachment = call(
|
|
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
|
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
|
|
)
|
|
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
|
|
status = None
|
|
while time.monotonic() - started < 60:
|
|
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
|
|
status = body['status']
|
|
if status in ('ready', 'failed'):
|
|
break
|
|
time.sleep(0.5)
|
|
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
|
|
|
|
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
|
|
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
|
|
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
|
|
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
|
|
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
|
|
continue
|
|
parent = b64hex(span.get('parentSpanId'))
|
|
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
|
|
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
|
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
|
|
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
|
|
|
|
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
|
|
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
|
|
streams = []
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
|
|
streams = (logs or {}).get('data', {}).get('result', [])
|
|
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
|
|
break
|
|
time.sleep(3)
|
|
print('\nLoki %s' % query)
|
|
lines = 0
|
|
for stream in streams:
|
|
labels = stream['stream']
|
|
for _, line in stream['values']:
|
|
lines += 1
|
|
print(' %-17s %-8s %-13s span_id=%s %s' % (
|
|
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
|
|
labels.get('span_id', ''), line[:60]))
|
|
|
|
kinds = {(service, span.get('kind')) for service, span in spans}
|
|
ok = (
|
|
status == 'ready'
|
|
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
|
|
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
|
|
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
|
|
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
|
|
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
|
|
)
|
|
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
|
|
sys.exit(0 if ok else 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|