127 lines
8.0 KiB
Python
127 lines
8.0 KiB
Python
"""Real OTLP metrics/logs and a W3C upload -> retry -> ready trace."""
|
|
import json
|
|
import atexit
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
import httpx
|
|
from stack import ROOT,docker,ready
|
|
|
|
OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True)
|
|
PROM=httpx.Client(base_url='http://127.0.0.1:9090',timeout=10)
|
|
atexit.register(PROM.close)
|
|
|
|
def sql(query): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',query)
|
|
def query(expression):
|
|
r=PROM.get('/api/v1/query',params={'query':expression})
|
|
r.raise_for_status(); data=r.json(); assert data['status']=='success',data
|
|
return data['data']['result']
|
|
|
|
def bootstrap(c):
|
|
username='otel_'+uuid.uuid4().hex[:20]; password='synthetic-otel-password'
|
|
r=c.post('/api/v1/users',json={'username':username,'display_name':'Telemetry fixture','password':password}); assert r.status_code==201,r.text
|
|
uid=r.json()['id']; r=c.post('/api/v1/auth/sessions',auth=(username,password)); assert r.status_code==201
|
|
token=r.json()['token']; headers={'Authorization':'Bearer '+token}
|
|
r=c.post('/api/v1/chats',headers=headers,json={'title':'Telemetry fixture','member_ids':[uid]}); assert r.status_code==201
|
|
return headers,r.json()['id'],token
|
|
|
|
def upload(c,headers,chat,extra=None):
|
|
r=c.post(f'/api/v1/chats/{chat}/attachments',params={'filename':'fixture.png'},headers={**headers,'Content-Type':'image/png',**(extra or {})},content=(ROOT/'tests/fixtures/sample.png').read_bytes())
|
|
assert r.status_code==202,r.text
|
|
return r
|
|
|
|
def wait_ready(c,headers,aid,timeout=140):
|
|
until=time.monotonic()+timeout
|
|
while time.monotonic()<until:
|
|
r=c.get('/api/v1/attachments/'+aid,headers=headers)
|
|
if r.status_code==200 and r.json()['status']=='ready': return r.json()
|
|
if r.status_code==200 and r.json()['status']=='failed': raise AssertionError(r.text)
|
|
time.sleep(.5)
|
|
raise AssertionError('Attachment not ready')
|
|
|
|
def trace_spans(data):
|
|
return [span for batch in data.get('batches',data.get('resourceSpans',[]))
|
|
for scope in batch.get('scopeSpans',batch.get('instrumentationLibrarySpans',[])) for span in scope.get('spans',[])]
|
|
|
|
def main():
|
|
result={}
|
|
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
|
|
ready(c); headers,chat,token=bootstrap(c)
|
|
trace_id=uuid.uuid4().hex; parent=uuid.uuid4().hex[:16]
|
|
docker('compose','stop','worker')
|
|
try:
|
|
r=upload(c,headers,chat,{'traceparent':f'00-{trace_id}-{parent}-01','tracestate':'research=mtusi'})
|
|
aid=r.json()['id']; request_id=r.headers['X-Request-Id']
|
|
job=json.loads(sql("SELECT row_to_json(j) FROM jobs j WHERE attachment_id='"+aid+"'"))
|
|
assert job['traceparent'].split('-')[1]==trace_id and job['tracestate']=='research=mtusi'
|
|
sql("INSERT INTO job_faults(job_id,point,remaining) VALUES ('"+job['id']+"','transient',1)")
|
|
finally: docker('compose','start','worker')
|
|
row=wait_ready(c,headers,aid)
|
|
assert len(row['variants'])==3
|
|
assert sql("SELECT staging_key IS NULL FROM attachments WHERE id='"+aid+"'")=='t'
|
|
deadline=time.monotonic()+90
|
|
while time.monotonic()<deadline:
|
|
response=httpx.get('http://127.0.0.1:3200/api/traces/'+trace_id,headers={'Accept':'application/json'},timeout=15)
|
|
if response.status_code==200:
|
|
trace_data=response.json(); spans=trace_spans(trace_data)
|
|
if sum(s['name']=='attachment.consume' for s in spans)>=2 and any(s['name']=='db.attachment.commit' for s in spans): break
|
|
time.sleep(2)
|
|
else: raise AssertionError('Upload/retry/ready trace not found in Tempo')
|
|
names=[s['name'] for s in spans]
|
|
assert names.count('POST /api/v1/chats/{chat_id}/attachments')==1,'Duplicate framework server spans'
|
|
for name in ('POST /api/v1/chats/{chat_id}/attachments','attachment.durable_accept','attachment.publish','attachment.consume','image.render','s3.put','db.attachment.commit'):
|
|
assert name in names,(name,names)
|
|
by_id={s['spanId']:s for s in spans}
|
|
consumers=[s for s in spans if s['name']=='attachment.consume']
|
|
assert all(by_id[s['parentSpanId']]['name']=='attachment.publish' for s in consumers)
|
|
assert all(s.get('traceState')=='research=mtusi' for s in consumers)
|
|
result['trace']={'trace_id':trace_id,'request_id':request_id,'attachment_id':aid,'job_id':job['id'],
|
|
'spans':len(spans),'consumer_attempts':len(consumers),'names':names,'tracestate_preserved':True,'staging_removed':True}
|
|
deadline=time.monotonic()+60
|
|
while time.monotonic()<deadline:
|
|
response=httpx.get('http://127.0.0.1:3100/loki/api/v1/query_range',params={
|
|
'query':'{service_name=~"messenger-(api|worker|dispatcher)"} |= "'+trace_id+'"',
|
|
'start':str(int((time.time()-600)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'},timeout=15)
|
|
response.raise_for_status(); logs=response.json()
|
|
lines=[json.loads(line) for stream in logs['data']['result'] for _,line in stream['values']]
|
|
outcomes={line.get('outcome') for line in lines if line.get('event')=='job_finished'}
|
|
if {'retry','ready'}<=outcomes and any(line.get('request_id')==request_id for line in lines): break
|
|
time.sleep(2)
|
|
else: raise AssertionError('Correlated request/retry/ready logs not found in Loki')
|
|
raw=json.dumps([trace_data,logs])
|
|
assert token not in raw and 'synthetic-otel-password' not in raw
|
|
result['logs']={'count':len(lines),'outcomes':sorted(x for x in outcomes if x),'request_id_correlated':True,'secret_values_absent':True}
|
|
(OUT/'upload-trace.json').write_text(json.dumps(trace_data,indent=2)+'\n',encoding='utf-8')
|
|
(OUT/'upload-logs.json').write_text(json.dumps(logs,indent=2)+'\n',encoding='utf-8')
|
|
required=['http_server_requests_total','http_server_duration_seconds_bucket','jobs_processed_total',
|
|
'job_processing_duration_seconds_bucket','attachment_time_to_ready_seconds_bucket',
|
|
'queue_depth','queue_oldest_age_seconds','db_pool_used','db_pool_waiting','worker_busy','worker_capacity',
|
|
'process_rss_bytes','process_cpu_time_seconds','attachment_slo_eligible','attachment_slo_bad']
|
|
deadline=time.monotonic()+30
|
|
while True:
|
|
values={metric:query(metric) for metric in required}
|
|
if all(values.values()): break
|
|
assert time.monotonic()<deadline,[metric for metric,rows in values.items() if not rows]
|
|
time.sleep(2)
|
|
for metric,rows in values.items():
|
|
for sample in rows:
|
|
assert not {'user_id','chat_id','message_id','attachment_id','request_id','filename','url'} & sample['metric'].keys()
|
|
if 'route' in sample['metric']: assert aid not in sample['metric']['route'] and chat not in sample['metric']['route']
|
|
(OUT/'metric-samples.json').write_text(json.dumps(values,indent=2)+'\n',encoding='utf-8')
|
|
result['metrics']={'present':required,'forbidden_labels_absent':True}
|
|
with httpx.Client(base_url='http://127.0.0.1:3000',auth=('admin',(ROOT/'secrets/grafana-password').read_text().strip()),timeout=20) as grafana:
|
|
dashboard=grafana.get('/api/dashboards/uid/mtusi-messenger'); dashboard.raise_for_status()
|
|
panels=dashboard.json()['dashboard']['panels']; assert len(panels)==17
|
|
sources=grafana.get('/api/datasources'); sources.raise_for_status()
|
|
health={}
|
|
for source in sources.json():
|
|
response=grafana.get('/api/datasources/uid/'+source['uid']+'/health'); response.raise_for_status()
|
|
health[source['uid']]=response.json()['status']; assert health[source['uid']]=='OK'
|
|
result['grafana']={'provisioned_panels':len(panels),'datasource_health':health}
|
|
result['passed']=True
|
|
(OUT/'telemetry.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(result,indent=2))
|
|
|
|
if __name__=='__main__': main()
|