forked from mtusi2026/mtusi2026lab6
95 lines
5.5 KiB
Python
95 lines
5.5 KiB
Python
"""Grade 4: real stored signals, W3C chain with retry, dashboard and privacy."""
|
|
import base64
|
|
import hashlib
|
|
import os
|
|
import time
|
|
import uuid
|
|
from obs_helpers import *
|
|
|
|
|
|
def normalized_id(value):
|
|
if not value:return ''
|
|
if all(c in '0123456789abcdef' for c in value.lower()) and len(value) in (16,32):return value.lower()
|
|
return base64.b64decode(value).hex()
|
|
|
|
|
|
def main():
|
|
client=Client(timeout=30)
|
|
actor,_,_,chat=client.bootstrap()
|
|
marker='PRIVATE-BODY-'+uuid.uuid4().hex
|
|
client.message(actor,chat,marker)
|
|
trace_id=uuid.uuid4().hex
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
try:
|
|
compose('stop','worker')
|
|
response=client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
|
|
actor=actor,raw=raw,headers={'Content-Type':'image/png','traceparent':'00-'+trace_id+'-0123456789abcdef-01','tracestate':'lab=synthetic'},schema='Attachment')
|
|
item=response['json'];request_id=response['headers']['X-Request-Id']
|
|
check(response['headers']['X-Trace-Id']==trace_id,'incoming W3C traceparent not honored')
|
|
compose('stop','s3');start_only('worker')
|
|
wait_for(lambda: psql("SELECT attempts>=1 AND state='queued' FROM attachment_jobs WHERE id='"+item['id']+"';")=='t','retry not observed',35)
|
|
compose('start','s3');ready(client)
|
|
wait_state(client,actor,item)
|
|
check(client.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=actor)['bytes']==raw,'original changed')
|
|
def linked():
|
|
data=trace_data(trace_id);found=spans(data)
|
|
names={s['name'] for s in found}
|
|
consumers=[s for s in found if s['name']=='attachment.consume']
|
|
if len(consumers)>=2 and {'attachment.publish','image.process','s3.put','db.update','db.commit'}.issubset(names):return data
|
|
return None
|
|
data=wait_for(linked,'complete upload/publish/retry/consume/result trace missing',60)
|
|
found=spans(data)
|
|
publishers={normalized_id(s['spanId']) for s in found if s['name']=='attachment.publish'}
|
|
consumers=[s for s in found if s['name']=='attachment.consume']
|
|
check(all(normalized_id(s.get('parentSpanId','')) in publishers for s in consumers),'consumer parent is not producer')
|
|
correlated=wait_for(lambda: [entry for entry in logs(trace_id) if entry.get('event')=='job.completed' and entry.get('outcome')=='success'], 'worker log missing',40)
|
|
all_logs=logs(trace_id)
|
|
events={entry['event'] for entry in all_logs}
|
|
check({'http.request','job.published','job.started','job.completed'}.issubset(events),'API/publisher/worker logs not correlated')
|
|
check(any(e.get('request_id')==request_id and e['event']=='http.request' for e in all_logs),'request id correlation missing')
|
|
check(correlated[0]['request_id']==request_id,'worker lost originating request id')
|
|
encoded=json.dumps(data)+json.dumps(all_logs)+json.dumps(logs(service='messenger-api'))
|
|
for secret in (marker,actor['token'],actor['registration']['password'],'private-canary.png'):
|
|
check(secret not in encoded,'sensitive payload/header/filename entered telemetry')
|
|
# Actual stored metric names, not assumptions from SDK naming.
|
|
expressions={
|
|
'http_requests':'sum(messenger_http_requests_total)',
|
|
'http_histogram':'sum(messenger_http_duration_seconds_count)',
|
|
'jobs_success':'sum(messenger_jobs_processed_total{outcome="success"})',
|
|
'jobs_retry':'sum(messenger_jobs_processed_total{outcome="retry"})',
|
|
'ready_histogram':'sum(messenger_time_to_ready_seconds_count)',
|
|
'queue':'max(messenger_queue_depth)',
|
|
'pool':'max(messenger_db_pool_capacity)',
|
|
'worker_capacity':'max(messenger_worker_capacity)',
|
|
'memory':'max(messenger_container_memory_bytes)',
|
|
}
|
|
values={}
|
|
for name,expression in expressions.items():
|
|
wait_for(lambda: prom(expression),'missing metric '+name,20)
|
|
values[name]=number(expression)
|
|
check(values['jobs_success']>0 and values['jobs_retry']>0,'worker outcomes absent')
|
|
metric_rows=prom('{__name__=~"messenger_.*"}')
|
|
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','path','job_id'}
|
|
check(not any(forbidden.intersection(row['metric']) for row in metric_rows),'high-cardinality metric labels')
|
|
password=(ROOT/'secrets/grafana_admin_password').read_text().strip()
|
|
code='''
|
|
import base64,json,sys,urllib.request
|
|
p=sys.stdin.read().strip();req=urllib.request.Request('http://grafana:3000/api/dashboards/uid/mtusi-messenger',headers={'Authorization':'Basic '+base64.b64encode(('admin:'+p).encode()).decode()})
|
|
with urllib.request.urlopen(req,timeout=10) as r:d=json.load(r)
|
|
print(json.dumps({'title':d['dashboard']['title'],'panels':len(d['dashboard']['panels'])}))
|
|
'''
|
|
dashboard=json.loads(compose('exec','-T','worker','python3','-c',code,data=password))
|
|
check(dashboard['panels']>=18,'provisioned dashboard missing')
|
|
dump_evidence('trace-example.json',data)
|
|
dump_evidence('correlated-logs.json',all_logs)
|
|
output=dict(status='PASS',trace_id=trace_id,request_id=request_id,consumer_attempt_spans=len(consumers),
|
|
events=sorted(events),metrics=values,dashboard=dashboard,privacy='PASS',low_cardinality='PASS')
|
|
dump_evidence('observability-grade4.json',output)
|
|
print('PASS:',json.dumps(output))
|
|
finally:
|
|
compose('start','s3','worker');ready(client)
|
|
|
|
|
|
if __name__=='__main__':
|
|
run(main)
|