Files
mtusi2026lab6/scripts/capture-incidents.py
T
2026-11-17 13:08:13 +03:00

44 lines
3.5 KiB
Python

#!/usr/bin/env python3
"""Passively save stored log/trace evidence during the running SLO experiment."""
import json,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
from observations import get
source=ROOT/'evidence/slo-experiment.json';target=ROOT/'evidence/incident-signals.json'
result={};errors=[]
def logs(query,start,end):
return get('loki','/loki/api/v1/query_range',{'query':query,'start':str(int(start*1e9)),'end':str(int(end*1e9)),'limit':100})['data']['result']
def summary(d):
rows=[]
for b in d.get('batches',[]):
resource={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',[]):
for s in scope.get('spans',[]):
rows.append({'service':resource.get('service.name'),'name':s['name'],'kind':s.get('kind'),'span_id':s.get('spanId'),'parent_span_id':s.get('parentSpanId'),'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,'status':s.get('status')})
return rows
end=time.time()+8000
while time.time()<end:
try:
x=json.loads(source.read_text());last=x['observations'][-1];phase=last['phase'];event=next(e for e in reversed(x['events']) if e['phase']==phase)
if phase.startswith('db_long_') and time.time()-event['at']>60 and phase not in result:
rows=logs('{service_name="messenger-api"} |= "http_error"',event['at'],time.time()-3)
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
if ids:
traceid=ids[-1];spans=summary(get('tempo','/api/traces/'+traceid))
if any(s['name'].startswith('postgresql ') and s['duration_ms']>1000 for s in spans):result[phase]={'captured':time.time(),'observed':last,'trace_id':traceid,'spans':spans,'error_log_events':sum(len(r['values']) for r in rows),'diagnosis':'DB wait dominates server span; HTTP 5xx, not image processing'}
if phase.startswith('worker_long_') and time.time()-event['at']>70 and phase not in result:
worker=logs('{service_name="messenger-worker"} |= "job_received"',event['at'],time.time()-3)
uploads=logs('{service_name="messenger-api"} |= "http_request" | route="/api/v1/chats/{chat_id}/attachments" | status="202"',event['at'],time.time()-3)
result[phase]={'captured':time.time(),'observed':last,'worker_received_events':sum(len(r['values']) for r in worker),'accepted_upload_events':sum(len(r['values']) for r in uploads),'diagnosis':'Queue age and ready burn increase while API accepts uploads and no worker consumption is observed'}
if phase.startswith('worker_recovery_') and phase not in result and time.time()-event['at']>20:
rows=logs('{service_name="messenger-worker"} |= "job_ready"',event['at'],time.time()-3)
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
if ids:
traceid=ids[0];spans=summary(get('tempo','/api/traces/'+traceid))
if spans:result[phase]={'captured':time.time(),'trace_id':traceid,'spans':spans,'diagnosis':'Worker resumes consumption and publishes S3/DB results'}
target.write_text(json.dumps({'evidence':result,'collection_errors':errors[-20:]},indent=2)+'\n')
if x.get('completed') is not None:break
except Exception as e:errors.append({'at':time.time(),'type':type(e).__name__})
time.sleep(5)
print('Saved incident observations:',len(result))