Files
mtusi2026lab6/scripts/verify_telemetry.py
T

127 lines
8.0 KiB
Python

"""Real OTLP metrics/logs and a W3C upload -> retry -> ready trace."""
import json
import atexit
from pathlib import Path
import subprocess
import time
import uuid
import httpx
from stack import ROOT,docker,ready
OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True)
PROM=httpx.Client(base_url='http://127.0.0.1:9090',timeout=10)
atexit.register(PROM.close)
def sql(query): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',query)
def query(expression):
r=PROM.get('/api/v1/query',params={'query':expression})
r.raise_for_status(); data=r.json(); assert data['status']=='success',data
return data['data']['result']
def bootstrap(c):
username='otel_'+uuid.uuid4().hex[:20]; password='synthetic-otel-password'
r=c.post('/api/v1/users',json={'username':username,'display_name':'Telemetry fixture','password':password}); assert r.status_code==201,r.text
uid=r.json()['id']; r=c.post('/api/v1/auth/sessions',auth=(username,password)); assert r.status_code==201
token=r.json()['token']; headers={'Authorization':'Bearer '+token}
r=c.post('/api/v1/chats',headers=headers,json={'title':'Telemetry fixture','member_ids':[uid]}); assert r.status_code==201
return headers,r.json()['id'],token
def upload(c,headers,chat,extra=None):
r=c.post(f'/api/v1/chats/{chat}/attachments',params={'filename':'fixture.png'},headers={**headers,'Content-Type':'image/png',**(extra or {})},content=(ROOT/'tests/fixtures/sample.png').read_bytes())
assert r.status_code==202,r.text
return r
def wait_ready(c,headers,aid,timeout=140):
until=time.monotonic()+timeout
while time.monotonic()<until:
r=c.get('/api/v1/attachments/'+aid,headers=headers)
if r.status_code==200 and r.json()['status']=='ready': return r.json()
if r.status_code==200 and r.json()['status']=='failed': raise AssertionError(r.text)
time.sleep(.5)
raise AssertionError('Attachment not ready')
def trace_spans(data):
return [span for batch in data.get('batches',data.get('resourceSpans',[]))
for scope in batch.get('scopeSpans',batch.get('instrumentationLibrarySpans',[])) for span in scope.get('spans',[])]
def main():
result={}
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
ready(c); headers,chat,token=bootstrap(c)
trace_id=uuid.uuid4().hex; parent=uuid.uuid4().hex[:16]
docker('compose','stop','worker')
try:
r=upload(c,headers,chat,{'traceparent':f'00-{trace_id}-{parent}-01','tracestate':'research=mtusi'})
aid=r.json()['id']; request_id=r.headers['X-Request-Id']
job=json.loads(sql("SELECT row_to_json(j) FROM jobs j WHERE attachment_id='"+aid+"'"))
assert job['traceparent'].split('-')[1]==trace_id and job['tracestate']=='research=mtusi'
sql("INSERT INTO job_faults(job_id,point,remaining) VALUES ('"+job['id']+"','transient',1)")
finally: docker('compose','start','worker')
row=wait_ready(c,headers,aid)
assert len(row['variants'])==3
assert sql("SELECT staging_key IS NULL FROM attachments WHERE id='"+aid+"'")=='t'
deadline=time.monotonic()+90
while time.monotonic()<deadline:
response=httpx.get('http://127.0.0.1:3200/api/traces/'+trace_id,headers={'Accept':'application/json'},timeout=15)
if response.status_code==200:
trace_data=response.json(); spans=trace_spans(trace_data)
if sum(s['name']=='attachment.consume' for s in spans)>=2 and any(s['name']=='db.attachment.commit' for s in spans): break
time.sleep(2)
else: raise AssertionError('Upload/retry/ready trace not found in Tempo')
names=[s['name'] for s in spans]
assert names.count('POST /api/v1/chats/{chat_id}/attachments')==1,'Duplicate framework server spans'
for name in ('POST /api/v1/chats/{chat_id}/attachments','attachment.durable_accept','attachment.publish','attachment.consume','image.render','s3.put','db.attachment.commit'):
assert name in names,(name,names)
by_id={s['spanId']:s for s in spans}
consumers=[s for s in spans if s['name']=='attachment.consume']
assert all(by_id[s['parentSpanId']]['name']=='attachment.publish' for s in consumers)
assert all(s.get('traceState')=='research=mtusi' for s in consumers)
result['trace']={'trace_id':trace_id,'request_id':request_id,'attachment_id':aid,'job_id':job['id'],
'spans':len(spans),'consumer_attempts':len(consumers),'names':names,'tracestate_preserved':True,'staging_removed':True}
deadline=time.monotonic()+60
while time.monotonic()<deadline:
response=httpx.get('http://127.0.0.1:3100/loki/api/v1/query_range',params={
'query':'{service_name=~"messenger-(api|worker|dispatcher)"} |= "'+trace_id+'"',
'start':str(int((time.time()-600)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'},timeout=15)
response.raise_for_status(); logs=response.json()
lines=[json.loads(line) for stream in logs['data']['result'] for _,line in stream['values']]
outcomes={line.get('outcome') for line in lines if line.get('event')=='job_finished'}
if {'retry','ready'}<=outcomes and any(line.get('request_id')==request_id for line in lines): break
time.sleep(2)
else: raise AssertionError('Correlated request/retry/ready logs not found in Loki')
raw=json.dumps([trace_data,logs])
assert token not in raw and 'synthetic-otel-password' not in raw
result['logs']={'count':len(lines),'outcomes':sorted(x for x in outcomes if x),'request_id_correlated':True,'secret_values_absent':True}
(OUT/'upload-trace.json').write_text(json.dumps(trace_data,indent=2)+'\n',encoding='utf-8')
(OUT/'upload-logs.json').write_text(json.dumps(logs,indent=2)+'\n',encoding='utf-8')
required=['http_server_requests_total','http_server_duration_seconds_bucket','jobs_processed_total',
'job_processing_duration_seconds_bucket','attachment_time_to_ready_seconds_bucket',
'queue_depth','queue_oldest_age_seconds','db_pool_used','db_pool_waiting','worker_busy','worker_capacity',
'process_rss_bytes','process_cpu_time_seconds','attachment_slo_eligible','attachment_slo_bad']
deadline=time.monotonic()+30
while True:
values={metric:query(metric) for metric in required}
if all(values.values()): break
assert time.monotonic()<deadline,[metric for metric,rows in values.items() if not rows]
time.sleep(2)
for metric,rows in values.items():
for sample in rows:
assert not {'user_id','chat_id','message_id','attachment_id','request_id','filename','url'} & sample['metric'].keys()
if 'route' in sample['metric']: assert aid not in sample['metric']['route'] and chat not in sample['metric']['route']
(OUT/'metric-samples.json').write_text(json.dumps(values,indent=2)+'\n',encoding='utf-8')
result['metrics']={'present':required,'forbidden_labels_absent':True}
with httpx.Client(base_url='http://127.0.0.1:3000',auth=('admin',(ROOT/'secrets/grafana-password').read_text().strip()),timeout=20) as grafana:
dashboard=grafana.get('/api/dashboards/uid/mtusi-messenger'); dashboard.raise_for_status()
panels=dashboard.json()['dashboard']['panels']; assert len(panels)==17
sources=grafana.get('/api/datasources'); sources.raise_for_status()
health={}
for source in sources.json():
response=grafana.get('/api/datasources/uid/'+source['uid']+'/health'); response.raise_for_status()
health[source['uid']]=response.json()['status']; assert health[source['uid']]=='OK'
result['grafana']={'provisioned_panels':len(panels),'datasource_health':health}
result['passed']=True
(OUT/'telemetry.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8')
print(json.dumps(result,indent=2))
if __name__=='__main__': main()