97 lines
7.2 KiB
Python
97 lines
7.2 KiB
Python
"""Assert evidence criteria; fail on missing observations instead of fabricating PASS."""
|
|
from pathlib import Path
|
|
import json
|
|
import base64
|
|
import re
|
|
import sys
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
E=ROOT/'evidence'
|
|
sys.path.insert(0,str(ROOT/'scripts'))
|
|
from observe_experiment import query,http,save
|
|
def load(name):return json.loads((E/name).read_text())
|
|
def spans(trace):
|
|
result=[]
|
|
for batch in trace.get('batches',[]):
|
|
attrs={a['key']:next(iter(a['value'].values())) for a in batch.get('resource',{}).get('attributes',[])}
|
|
for scope in batch.get('scopeSpans',[]):
|
|
for span in scope.get('spans',[]):
|
|
result.append({**span,'service':attrs.get('service.name')})
|
|
return result
|
|
def main():
|
|
checks={}
|
|
def check(name,condition):
|
|
checks[name]=bool(condition)
|
|
if not condition:print('FAIL:',name)
|
|
result=load('slo_result.json')
|
|
check('experiment_completed',result['status']=='PASS')
|
|
check('two_repetitions',result['repetitions']==2)
|
|
check('short_spikes_do_not_page',not result['false_page_samples'])
|
|
for repeat in [1,2]:
|
|
for name in ['QueueLagGrowing','ReadyErrorBudgetBurn','APIHighErrorRate','APIErrorBudgetBurnFast','APIErrorBudgetBurnSlow']:
|
|
rows=[r for r in result['detections'] if r['incident'].startswith(str(repeat)+'_') and r['alert']==name and 'short_spike' not in r['incident']]
|
|
check(str(repeat)+'_'+name+'_firing_and_resolved',bool(rows) and all(r['detection_seconds'] is not None and r['resolved_time'] is not None and r.get('recovery_seconds') is not None for r in rows))
|
|
signals=load('signals_worker_'+str(repeat)+'.json')
|
|
graphs=[spans(t) for t in signals['traces'].values()]
|
|
chains=[]
|
|
for graph in graphs:
|
|
by_id={s['spanId']:s for s in graph}
|
|
consumers=[s for s in graph if s['kind']=='SPAN_KIND_CONSUMER']
|
|
producers=[s for s in graph if s['kind']=='SPAN_KIND_PRODUCER']
|
|
server=[s for s in graph if s['kind']=='SPAN_KIND_SERVER']
|
|
chain=bool(server and producers and consumers)
|
|
chain=chain and all(s.get('parentSpanId') in by_id and by_id[s['parentSpanId']]['kind']=='SPAN_KIND_PRODUCER' for s in consumers)
|
|
chain=chain and all(s.get('parentSpanId') in by_id and by_id[s['parentSpanId']]['kind']=='SPAN_KIND_SERVER' for s in producers)
|
|
chain=chain and any(s['name'].startswith('s3.') for s in graph) and any(s['name']=='db.commit' for s in graph)
|
|
if chain:chains.append(graph)
|
|
check(str(repeat)+'_upload_producer_consumer_db_s3_chain',bool(chains))
|
|
|
|
api_entries=[json.loads(value[1]) for stream in signals.get('correlated_api_logs',signals['api_logs'])['data']['result'] for value in stream['values']]
|
|
worker_entries=[json.loads(value[1]) for stream in signals['worker_logs']['data']['result'] for value in stream['values']]
|
|
chain_ids={base64.b64decode(g[0]['traceId']).hex() for g in chains}
|
|
check(str(repeat)+'_api_and_worker_logs',bool(api_entries) and bool(worker_entries))
|
|
check(str(repeat)+'_correlated_http_worker_logs',any(e.get('trace_id') in chain_ids and e.get('request_id') for e in api_entries) and any(e.get('trace_id') in chain_ids and e.get('job_id') for e in worker_entries))
|
|
db=load('signals_db_'+str(repeat)+'.json')
|
|
dbspans=[s for t in db['traces'].values() for s in spans(t)]
|
|
check(str(repeat)+'_db_error_trace',any(s['name']=='db.execute' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' and any(a['key']=='error.type' and a['value'].get('stringValue')=='LockNotAvailable' for a in s.get('attributes',[])) and (int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e9>=1.8 for s in dbspans))
|
|
retry=load('retry_trace_result.json')
|
|
graphs=[spans(t) for t in load('signals_retry.json')['traces'].values()]
|
|
check('verified_retry_job_ready',retry['status']=='PASS' and retry['final_job']['status']=='ready' and retry['final_job']['attempts']>=2)
|
|
check('real_retry_in_same_trace',any(sum(s['kind']=='SPAN_KIND_CONSUMER' for s in g)>=2 and
|
|
any(s['kind']=='SPAN_KIND_SERVER' for s in g) and any(s['name']=='s3.get' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in g) for g in graphs))
|
|
rows=load('slo_requests.json')
|
|
check('baseline_and_backend_messages_succeed',all(r['status']==201 for r in rows if r['phase'] in ['baseline','backend_down']))
|
|
outage=load('backend_outage.json')
|
|
def value(d,q):
|
|
rows=d[q];return float(rows[0]['value'][1]) if rows else 0
|
|
q='sum(otelcol_exporter_send_failed_spans_total)'
|
|
check('backend_failed_exports_observed',value(outage['during'],q)>value(outage['before'],q))
|
|
samples=load('slo_samples.json')
|
|
# Docker stats verifies actual limit/usage; a below-limit snapshot is bounded, not a leak proof.
|
|
backend=[s for s in samples if s['phase']=='backend_down' and s.get('resources')]
|
|
check('backend_resource_samples',len(backend)>=3)
|
|
def parse_bytes(text):
|
|
number,unit=re.fullmatch(r'([0-9.]+)\s*([A-Za-z]+)',text.strip()).groups()
|
|
return float(number)*{'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3,'kB':1000,'MB':1000**2,'GB':1000**3}[unit]
|
|
collectors=[json.loads(line) for s in backend for line in s['resources'].splitlines() if line and json.loads(line).get('Name','').startswith('mtusi2026lab6-collector-')]
|
|
memory=[parse_bytes(r['MemUsage'].split('/')[0]) for r in collectors]
|
|
check('collector_memory_below_256_MiB',bool(memory) and max(memory)<256*1024**2)
|
|
save('backend_memory.json',{'samples':len(memory),'min_bytes':min(memory) if memory else None,'max_bytes':max(memory) if memory else None,
|
|
'first_bytes':memory[0] if memory else None,'last_bytes':memory[-1] if memory else None,'limit_bytes':256*1024**2})
|
|
required=['app_http_requests_total','app_http_duration_seconds_bucket','jobs_processed_total','job_duration_seconds_bucket',
|
|
'queue_depth','queue_oldest_age_seconds','db_pool_used','db_pool_waiting','worker_busy','worker_capacity',
|
|
'process_cpu_seconds','process_memory_bytes','attachment_ready_seconds_bucket','ready_deadline_jobs_total','ready_deadline_missed_total']
|
|
names=http('http://localhost:9096/api/v1/label/__name__/values')['data']
|
|
check('all_signal_metrics_exist',all(n in names for n in required))
|
|
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','text'}
|
|
series=http('http://localhost:9096/api/v1/series?match%5B%5D=app_http_requests_total')['data']
|
|
check('no_forbidden_metric_labels',not any(forbidden.intersection(s) for s in series))
|
|
dashboard=load_dashboard()
|
|
check('dashboard_has_real_queries',all(query(t['expr']) for p in dashboard['panels'] for t in p.get('targets',[]) if 'otelcol_exporter_enqueue_failed' not in t['expr'] and 'ALERTS' not in t['expr']))
|
|
check('grafana_healthy',http('http://localhost:3006/api/health')['database']=='ok')
|
|
save('observability_audit.json',{'status':'PASS' if all(checks.values()) else 'FAIL','checks':checks})
|
|
print(json.dumps(checks,indent=2))
|
|
assert all(checks.values()),'Evidence criteria incomplete'
|
|
def load_dashboard():
|
|
return json.loads((ROOT/'infra/otel/grafana/dashboards/lab6.json').read_text())
|
|
if __name__=='__main__':main()
|