Files
mtusi2026lab6/student-tests/verify_observability.py
T

95 lines
5.5 KiB
Python

"""Grade 4: real stored signals, W3C chain with retry, dashboard and privacy."""
import base64
import hashlib
import os
import time
import uuid
from obs_helpers import *
def normalized_id(value):
if not value:return ''
if all(c in '0123456789abcdef' for c in value.lower()) and len(value) in (16,32):return value.lower()
return base64.b64decode(value).hex()
def main():
client=Client(timeout=30)
actor,_,_,chat=client.bootstrap()
marker='PRIVATE-BODY-'+uuid.uuid4().hex
client.message(actor,chat,marker)
trace_id=uuid.uuid4().hex
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
try:
compose('stop','worker')
response=client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
actor=actor,raw=raw,headers={'Content-Type':'image/png','traceparent':'00-'+trace_id+'-0123456789abcdef-01','tracestate':'lab=synthetic'},schema='Attachment')
item=response['json'];request_id=response['headers']['X-Request-Id']
check(response['headers']['X-Trace-Id']==trace_id,'incoming W3C traceparent not honored')
compose('stop','s3');start_only('worker')
wait_for(lambda: psql("SELECT attempts>=1 AND state='queued' FROM attachment_jobs WHERE id='"+item['id']+"';")=='t','retry not observed',35)
compose('start','s3');ready(client)
wait_state(client,actor,item)
check(client.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=actor)['bytes']==raw,'original changed')
def linked():
data=trace_data(trace_id);found=spans(data)
names={s['name'] for s in found}
consumers=[s for s in found if s['name']=='attachment.consume']
if len(consumers)>=2 and {'attachment.publish','image.process','s3.put','db.update','db.commit'}.issubset(names):return data
return None
data=wait_for(linked,'complete upload/publish/retry/consume/result trace missing',60)
found=spans(data)
publishers={normalized_id(s['spanId']) for s in found if s['name']=='attachment.publish'}
consumers=[s for s in found if s['name']=='attachment.consume']
check(all(normalized_id(s.get('parentSpanId','')) in publishers for s in consumers),'consumer parent is not producer')
correlated=wait_for(lambda: [entry for entry in logs(trace_id) if entry.get('event')=='job.completed' and entry.get('outcome')=='success'], 'worker log missing',40)
all_logs=logs(trace_id)
events={entry['event'] for entry in all_logs}
check({'http.request','job.published','job.started','job.completed'}.issubset(events),'API/publisher/worker logs not correlated')
check(any(e.get('request_id')==request_id and e['event']=='http.request' for e in all_logs),'request id correlation missing')
check(correlated[0]['request_id']==request_id,'worker lost originating request id')
encoded=json.dumps(data)+json.dumps(all_logs)+json.dumps(logs(service='messenger-api'))
for secret in (marker,actor['token'],actor['registration']['password'],'private-canary.png'):
check(secret not in encoded,'sensitive payload/header/filename entered telemetry')
# Actual stored metric names, not assumptions from SDK naming.
expressions={
'http_requests':'sum(messenger_http_requests_total)',
'http_histogram':'sum(messenger_http_duration_seconds_count)',
'jobs_success':'sum(messenger_jobs_processed_total{outcome="success"})',
'jobs_retry':'sum(messenger_jobs_processed_total{outcome="retry"})',
'ready_histogram':'sum(messenger_time_to_ready_seconds_count)',
'queue':'max(messenger_queue_depth)',
'pool':'max(messenger_db_pool_capacity)',
'worker_capacity':'max(messenger_worker_capacity)',
'memory':'max(messenger_container_memory_bytes)',
}
values={}
for name,expression in expressions.items():
wait_for(lambda: prom(expression),'missing metric '+name,20)
values[name]=number(expression)
check(values['jobs_success']>0 and values['jobs_retry']>0,'worker outcomes absent')
metric_rows=prom('{__name__=~"messenger_.*"}')
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','path','job_id'}
check(not any(forbidden.intersection(row['metric']) for row in metric_rows),'high-cardinality metric labels')
password=(ROOT/'secrets/grafana_admin_password').read_text().strip()
code='''
import base64,json,sys,urllib.request
p=sys.stdin.read().strip();req=urllib.request.Request('http://grafana:3000/api/dashboards/uid/mtusi-messenger',headers={'Authorization':'Basic '+base64.b64encode(('admin:'+p).encode()).decode()})
with urllib.request.urlopen(req,timeout=10) as r:d=json.load(r)
print(json.dumps({'title':d['dashboard']['title'],'panels':len(d['dashboard']['panels'])}))
'''
dashboard=json.loads(compose('exec','-T','worker','python3','-c',code,data=password))
check(dashboard['panels']>=18,'provisioned dashboard missing')
dump_evidence('trace-example.json',data)
dump_evidence('correlated-logs.json',all_logs)
output=dict(status='PASS',trace_id=trace_id,request_id=request_id,consumer_attempt_spans=len(consumers),
events=sorted(events),metrics=values,dashboard=dashboard,privacy='PASS',low_cardinality='PASS')
dump_evidence('observability-grade4.json',output)
print('PASS:',json.dumps(output))
finally:
compose('start','s3','worker');ready(client)
if __name__=='__main__':
run(main)