Files
mtusi2026lab6/student-tests/observability.py
T

82 lines
3.6 KiB
Python

#!/usr/bin/env python3
"""Assert actual application signals in each backend; persist no credentials."""
import json
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
ROOT = Path(__file__).resolve().parents[1]
def get(base, path, params=None):
if params:
path += '?' + urllib.parse.urlencode(params)
with urllib.request.urlopen(base + path, timeout=15) as response:
return json.load(response)
def eventually(fn, seconds=90):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
value = fn()
if value:
return value
except Exception as exc:
last = type(exc).__name__
time.sleep(2)
raise AssertionError('signal did not arrive: ' + str(last))
def logs(service, suffix=''):
return get('http://localhost:3100', '/loki/api/v1/query_range', {
'query': '{service_name="'+service+'"}' + suffix,
'start': int((time.time()-600)*1e9), 'end': int(time.time()*1e9), 'limit': 100
})['data']['result']
def trace_exists(tid):
data = get('http://localhost:3200', '/api/traces/' + tid)
return data if data.get('batches') or data.get('resourceSpans') else None
def main():
c = Client('http://localhost:18086')
alice, bob, eve, chat = c.bootstrap()
response = c.message(alice, chat, 'telemetry synthetic check')
# Follow a controlled request by server-generated X-Trace-Id and X-Request-Id.
response = c.call('GET', '/api/v1/chats/'+chat['id']+'/messages', actor=alice)
headers = response['headers']
tid = headers.get('X-Trace-Id')
rid = headers.get('X-Request-Id')
check(bool(tid) and len(tid)==32, 'missing trace ID')
job_started = time.time()
att, _ = c.attachment(alice, chat)
c.wait_ready(alice, att)
api_logs = eventually(lambda: logs('messenger-api', ' | trace_id="'+tid+'"'))
api_trace = eventually(lambda: trace_exists(tid))
worker_logs = eventually(lambda: [row for row in logs('messenger-worker', ' | event="job.success"') if int(row['values'][0][0])/1e9 >= job_started])
values = worker_logs[0]['values']
# Native OTLP metadata carries trace_id without making it a metric label.
worker_tid = values[0][2].get('trace_id') if len(values[0]) > 2 else None
if not worker_tid:
worker_tid = worker_logs[0]['stream'].get('trace_id')
check(bool(worker_tid), 'worker log lacks trace ID')
worker_trace = eventually(lambda: trace_exists(worker_tid))
metric_names = ['api_requests_total','api_duration_seconds_bucket','worker_jobs_total','worker_duration_seconds_bucket']
metric_results = {}
for name in metric_names:
metric_results[name] = eventually(lambda: get('http://localhost:19091','/api/v1/query',{'query': name})['data']['result'])
for results in metric_results.values():
for result in results:
check(not (set(result['metric']) & {'user_id','chat_id','message_id','attachment_id','request_id','filename'}), 'high cardinality metric labels')
proof = {'api_trace_id':tid,'api_request_id':rid,'worker_trace_id':worker_tid,
'api_logs':api_logs,'worker_logs':worker_logs,'api_trace':api_trace,'worker_trace':worker_trace,
'metrics':metric_results}
(ROOT/'evidence'/'telemetry.json').write_text(json.dumps(proof,ensure_ascii=False,indent=2)+'\n')
print('PASS: API trace + matching log, worker consumer trace + matching log, four real SDK metrics; evidence/telemetry.json')
if __name__ == '__main__':
main()