Files
mtusi2026lab6/scripts/audit_observability.py
2026-10-30 21:38:40 +03:00

97 lines
7.2 KiB
Python

"""Assert evidence criteria; fail on missing observations instead of fabricating PASS."""
from pathlib import Path
import json
import base64
import re
import sys
ROOT=Path(__file__).resolve().parents[1]
E=ROOT/'evidence'
sys.path.insert(0,str(ROOT/'scripts'))
from observe_experiment import query,http,save
def load(name):return json.loads((E/name).read_text())
def spans(trace):
result=[]
for batch in trace.get('batches',[]):
attrs={a['key']:next(iter(a['value'].values())) for a in batch.get('resource',{}).get('attributes',[])}
for scope in batch.get('scopeSpans',[]):
for span in scope.get('spans',[]):
result.append({**span,'service':attrs.get('service.name')})
return result
def main():
checks={}
def check(name,condition):
checks[name]=bool(condition)
if not condition:print('FAIL:',name)
result=load('slo_result.json')
check('experiment_completed',result['status']=='PASS')
check('two_repetitions',result['repetitions']==2)
check('short_spikes_do_not_page',not result['false_page_samples'])
for repeat in [1,2]:
for name in ['QueueLagGrowing','ReadyErrorBudgetBurn','APIHighErrorRate','APIErrorBudgetBurnFast','APIErrorBudgetBurnSlow']:
rows=[r for r in result['detections'] if r['incident'].startswith(str(repeat)+'_') and r['alert']==name and 'short_spike' not in r['incident']]
check(str(repeat)+'_'+name+'_firing_and_resolved',bool(rows) and all(r['detection_seconds'] is not None and r['resolved_time'] is not None and r.get('recovery_seconds') is not None for r in rows))
signals=load('signals_worker_'+str(repeat)+'.json')
graphs=[spans(t) for t in signals['traces'].values()]
chains=[]
for graph in graphs:
by_id={s['spanId']:s for s in graph}
consumers=[s for s in graph if s['kind']=='SPAN_KIND_CONSUMER']
producers=[s for s in graph if s['kind']=='SPAN_KIND_PRODUCER']
server=[s for s in graph if s['kind']=='SPAN_KIND_SERVER']
chain=bool(server and producers and consumers)
chain=chain and all(s.get('parentSpanId') in by_id and by_id[s['parentSpanId']]['kind']=='SPAN_KIND_PRODUCER' for s in consumers)
chain=chain and all(s.get('parentSpanId') in by_id and by_id[s['parentSpanId']]['kind']=='SPAN_KIND_SERVER' for s in producers)
chain=chain and any(s['name'].startswith('s3.') for s in graph) and any(s['name']=='db.commit' for s in graph)
if chain:chains.append(graph)
check(str(repeat)+'_upload_producer_consumer_db_s3_chain',bool(chains))
api_entries=[json.loads(value[1]) for stream in signals.get('correlated_api_logs',signals['api_logs'])['data']['result'] for value in stream['values']]
worker_entries=[json.loads(value[1]) for stream in signals['worker_logs']['data']['result'] for value in stream['values']]
chain_ids={base64.b64decode(g[0]['traceId']).hex() for g in chains}
check(str(repeat)+'_api_and_worker_logs',bool(api_entries) and bool(worker_entries))
check(str(repeat)+'_correlated_http_worker_logs',any(e.get('trace_id') in chain_ids and e.get('request_id') for e in api_entries) and any(e.get('trace_id') in chain_ids and e.get('job_id') for e in worker_entries))
db=load('signals_db_'+str(repeat)+'.json')
dbspans=[s for t in db['traces'].values() for s in spans(t)]
check(str(repeat)+'_db_error_trace',any(s['name']=='db.execute' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' and any(a['key']=='error.type' and a['value'].get('stringValue')=='LockNotAvailable' for a in s.get('attributes',[])) and (int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e9>=1.8 for s in dbspans))
retry=load('retry_trace_result.json')
graphs=[spans(t) for t in load('signals_retry.json')['traces'].values()]
check('verified_retry_job_ready',retry['status']=='PASS' and retry['final_job']['status']=='ready' and retry['final_job']['attempts']>=2)
check('real_retry_in_same_trace',any(sum(s['kind']=='SPAN_KIND_CONSUMER' for s in g)>=2 and
any(s['kind']=='SPAN_KIND_SERVER' for s in g) and any(s['name']=='s3.get' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in g) for g in graphs))
rows=load('slo_requests.json')
check('baseline_and_backend_messages_succeed',all(r['status']==201 for r in rows if r['phase'] in ['baseline','backend_down']))
outage=load('backend_outage.json')
def value(d,q):
rows=d[q];return float(rows[0]['value'][1]) if rows else 0
q='sum(otelcol_exporter_send_failed_spans_total)'
check('backend_failed_exports_observed',value(outage['during'],q)>value(outage['before'],q))
samples=load('slo_samples.json')
# Docker stats verifies actual limit/usage; a below-limit snapshot is bounded, not a leak proof.
backend=[s for s in samples if s['phase']=='backend_down' and s.get('resources')]
check('backend_resource_samples',len(backend)>=3)
def parse_bytes(text):
number,unit=re.fullmatch(r'([0-9.]+)\s*([A-Za-z]+)',text.strip()).groups()
return float(number)*{'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3,'kB':1000,'MB':1000**2,'GB':1000**3}[unit]
collectors=[json.loads(line) for s in backend for line in s['resources'].splitlines() if line and json.loads(line).get('Name','').startswith('mtusi2026lab6-collector-')]
memory=[parse_bytes(r['MemUsage'].split('/')[0]) for r in collectors]
check('collector_memory_below_256_MiB',bool(memory) and max(memory)<256*1024**2)
save('backend_memory.json',{'samples':len(memory),'min_bytes':min(memory) if memory else None,'max_bytes':max(memory) if memory else None,
'first_bytes':memory[0] if memory else None,'last_bytes':memory[-1] if memory else None,'limit_bytes':256*1024**2})
required=['app_http_requests_total','app_http_duration_seconds_bucket','jobs_processed_total','job_duration_seconds_bucket',
'queue_depth','queue_oldest_age_seconds','db_pool_used','db_pool_waiting','worker_busy','worker_capacity',
'process_cpu_seconds','process_memory_bytes','attachment_ready_seconds_bucket','ready_deadline_jobs_total','ready_deadline_missed_total']
names=http('http://localhost:9096/api/v1/label/__name__/values')['data']
check('all_signal_metrics_exist',all(n in names for n in required))
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','text'}
series=http('http://localhost:9096/api/v1/series?match%5B%5D=app_http_requests_total')['data']
check('no_forbidden_metric_labels',not any(forbidden.intersection(s) for s in series))
dashboard=load_dashboard()
check('dashboard_has_real_queries',all(query(t['expr']) for p in dashboard['panels'] for t in p.get('targets',[]) if 'otelcol_exporter_enqueue_failed' not in t['expr'] and 'ALERTS' not in t['expr']))
check('grafana_healthy',http('http://localhost:3006/api/health')['database']=='ok')
save('observability_audit.json',{'status':'PASS' if all(checks.values()) else 'FAIL','checks':checks})
print(json.dumps(checks,indent=2))
assert all(checks.values()),'Evidence criteria incomplete'
def load_dashboard():
return json.loads((ROOT/'infra/otel/grafana/dashboards/lab6.json').read_text())
if __name__=='__main__':main()