forked from mtusi2026/mtusi2026lab6
44 lines
3.5 KiB
Python
44 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Passively save stored log/trace evidence during the running SLO experiment."""
|
|
import json,sys,time
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
|
|
from observations import get
|
|
source=ROOT/'evidence/slo-experiment.json';target=ROOT/'evidence/incident-signals.json'
|
|
result={};errors=[]
|
|
def logs(query,start,end):
|
|
return get('loki','/loki/api/v1/query_range',{'query':query,'start':str(int(start*1e9)),'end':str(int(end*1e9)),'limit':100})['data']['result']
|
|
def summary(d):
|
|
rows=[]
|
|
for b in d.get('batches',[]):
|
|
resource={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
|
|
for scope in b.get('scopeSpans',[]):
|
|
for s in scope.get('spans',[]):
|
|
rows.append({'service':resource.get('service.name'),'name':s['name'],'kind':s.get('kind'),'span_id':s.get('spanId'),'parent_span_id':s.get('parentSpanId'),'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,'status':s.get('status')})
|
|
return rows
|
|
end=time.time()+8000
|
|
while time.time()<end:
|
|
try:
|
|
x=json.loads(source.read_text());last=x['observations'][-1];phase=last['phase'];event=next(e for e in reversed(x['events']) if e['phase']==phase)
|
|
if phase.startswith('db_long_') and time.time()-event['at']>60 and phase not in result:
|
|
rows=logs('{service_name="messenger-api"} |= "http_error"',event['at'],time.time()-3)
|
|
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
|
|
if ids:
|
|
traceid=ids[-1];spans=summary(get('tempo','/api/traces/'+traceid))
|
|
if any(s['name'].startswith('postgresql ') and s['duration_ms']>1000 for s in spans):result[phase]={'captured':time.time(),'observed':last,'trace_id':traceid,'spans':spans,'error_log_events':sum(len(r['values']) for r in rows),'diagnosis':'DB wait dominates server span; HTTP 5xx, not image processing'}
|
|
if phase.startswith('worker_long_') and time.time()-event['at']>70 and phase not in result:
|
|
worker=logs('{service_name="messenger-worker"} |= "job_received"',event['at'],time.time()-3)
|
|
uploads=logs('{service_name="messenger-api"} |= "http_request" | route="/api/v1/chats/{chat_id}/attachments" | status="202"',event['at'],time.time()-3)
|
|
result[phase]={'captured':time.time(),'observed':last,'worker_received_events':sum(len(r['values']) for r in worker),'accepted_upload_events':sum(len(r['values']) for r in uploads),'diagnosis':'Queue age and ready burn increase while API accepts uploads and no worker consumption is observed'}
|
|
if phase.startswith('worker_recovery_') and phase not in result and time.time()-event['at']>20:
|
|
rows=logs('{service_name="messenger-worker"} |= "job_ready"',event['at'],time.time()-3)
|
|
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
|
|
if ids:
|
|
traceid=ids[0];spans=summary(get('tempo','/api/traces/'+traceid))
|
|
if spans:result[phase]={'captured':time.time(),'trace_id':traceid,'spans':spans,'diagnosis':'Worker resumes consumption and publishes S3/DB results'}
|
|
target.write_text(json.dumps({'evidence':result,'collection_errors':errors[-20:]},indent=2)+'\n')
|
|
if x.get('completed') is not None:break
|
|
except Exception as e:errors.append({'at':time.time(),'type':type(e).__name__})
|
|
time.sleep(5)
|
|
print('Saved incident observations:',len(result))
|