Files
mtusi2026lab6/tests/observability_live.py
T

342 lines
19 KiB
Python

#!/usr/bin/env python3
"""Lab6 live evidence. Only synthetic test stack; no volume removal.
Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads.
"""
import argparse
import base64
import math
import concurrent.futures
import json
import os
import subprocess
import time
import urllib.parse
import urllib.request
import urllib.error
from pathlib import Path
from client import Client, Failure, check, run
from pipeline_extended import compose, pg, scalar, wait_for
from pipeline_resilience import verify_ready
ROOT=Path(__file__).resolve().parents[1]
PROM='http://127.0.0.1:9090'
LOKI='http://127.0.0.1:3100'
TEMPO='http://127.0.0.1:3200'
GRAFANA='http://127.0.0.1:3000'
ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)'
LAG='max(lab_queue_oldest_age_seconds)'
DEPTH='max(lab_queue_depth)'
FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
# Explicit Prometheus readers can append unit suffixes to Collector self-metrics.
# Missing gauges are not zero: absent telemetry must fail the outage test.
QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})'
QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})'
RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})'
def get(url, params=None, *, as_json=True):
try:
with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r:
raw=r.read(32*1024*1024)
return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace')
except (urllib.error.URLError,TimeoutError,ValueError) as exc:
raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None
def prom(expr):
data=get(PROM+'/api/v1/query',{'query':expr})
check(data.get('status')=='success','PromQL query failed')
return data['data']['result']
def val(expr):
rows=prom(expr)
return float(rows[0]['value'][1]) if rows else None
def alert_state(name):
rules=get(PROM+'/api/v1/rules')['data']['groups']
for group in rules:
for rule in group['rules']:
if rule['name']==name:
check(rule.get('health')=='ok','alert rule is not healthy: '+name)
return rule['state']
raise Failure('alert rule missing: '+name)
def wait_alert(name,state,seconds=100):
return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2)
def ready():
for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'):
wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2)
wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100)
wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40)
wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40)
def trace_spans(tid):
raw=get(TEMPO+'/api/traces/'+tid)
spans=[]
def walk(obj):
if isinstance(obj,dict):
if {'spanId','traceId','name'}<=set(obj): spans.append(obj)
else:
for v in obj.values():walk(v)
elif isinstance(obj,list):
for v in obj:walk(v)
walk(raw)
return spans
def logs(tid,service,request_id=None):
expr='{service_name="%s"} | trace_id="%s"' % (service,tid)
if request_id: expr+=' | request_id="%s"'%request_id
data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100})
return data['data']['result']
def upload(c,alice,chat):
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
tid=r['headers'].get('X-Trace-Id','')
check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement')
return r['json'],raw,tid,r['headers']['X-Request-Id']
def span_attrs(s):
attrs={}
for a in s.get('attributes',[]):
value=a.get('value',{})
attrs[a['key']]=next(iter(value.values()),None)
return attrs
def chain_ok(spans):
names={s['name'] for s in spans}
required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'}
if not required<=names:return False
producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'}
consumers=[s for s in spans if s['name']=='rabbitmq.consume']
outcomes={span_attrs(s).get('job.outcome') for s in consumers}
return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers)
def trace_retry(c):
alice,bob,_,chat=c.bootstrap()
compose('stop','worker')
try:
att,raw,tid,rid=upload(c,alice,chat)
# Override only worker, never a public endpoint; reset in finally.
env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'}
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180)
verify_ready(c,bob,att,raw,seconds=180)
spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3)
wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
def worker_events(streams):
return {v[1] for row in streams for v in row.get('values',[])}
wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90)
# Keep only allowlisted names/IDs, not arbitrary span values or log payloads.
summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans]
result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True}
print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid)
return result
finally:
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180)
def queue_lag(c):
alice,bob,_,chat=c.bootstrap()
wait_alert('LabQueueLag','inactive')
compose('stop','worker')
try:
att,raw,tid,rid=upload(c,alice,chat)
t=time.monotonic()
wait_alert('LabQueueLag','firing')
lag=val(LAG);depth=val(DEPTH)
check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics')
detected=time.monotonic()-t
print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True)
compose('up','-d','worker')
verify_ready(c,bob,att,raw,seconds=180)
wait_alert('LabQueueLag','inactive',seconds=120)
wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2)
print('PASS queue-lag: firing -> ready -> resolved')
return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)}
finally:compose('up','-d','worker')
def db_error(c):
alice,_,_,_=c.bootstrap()
wait_alert('LabAPIHighErrorRate','inactive')
user=compose('exec','-T','postgres','printenv','POSTGRES_USER')
db=compose('exec','-T','postgres','printenv','POSTGRES_DB')
# Bounded, isolated table lock. Health SELECT 1 works, authenticated business
# read times out at statement_timeout. Never stop DB (LB would hide API 5xx).
cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;']
lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15)
start=time.monotonic(); traces=[]; codes=[]
while time.monotonic()-start < 43:
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
def get_user(_):
r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User')
return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id')
for code,tid,rid in pool.map(get_user,range(3)):
codes.append(code)
if code==503 and tid:traces.append((tid,rid))
if traces and alert_state('LabAPIHighErrorRate')=='firing':break
check(bool(traces),'no real HTTP 503 from DB lock')
wait_alert('LabAPIHighErrorRate','firing',seconds=20)
ratio=val(ERR)
check(ratio is not None and ratio>0.05,'error ratio below threshold')
tid,rid=traces[0]
spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2)
wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60)
print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True)
finally:
# SQL transaction has server-side 60s bound; wait for normal COMMIT.
try:lock.wait(timeout=75)
except subprocess.TimeoutExpired:
lock.terminate();lock.wait(timeout=10)
# Keep successful business traffic in the evaluation window to prove recovery.
t=time.monotonic()
while time.monotonic()-t<65:
c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break
time.sleep(1)
wait_alert('LabAPIHighErrorRate','inactive',seconds=40)
print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved')
return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]}
def collector_sample():
check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down')
rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY)
check(rss is not None and math.isfinite(rss) and rss>0,
'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus')
check(queue is not None and math.isfinite(queue) and queue>=0,
'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics')
check(capacity is not None and math.isfinite(capacity) and capacity==64,
'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity))
return rss,queue,capacity
def backend_down(c):
alice,bob,_,chat=c.bootstrap()
baseline=val(FAILED) or 0
cid=compose('ps','-q','otel-collector')
before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip())
check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit))
rss,queue,capacity=collector_sample()
print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True)
compose('stop','tempo')
try:
start=time.monotonic();n=0;memory=[];queues=[];failures=[]
att,raw,tid,_=upload(c,alice,chat)
while time.monotonic()-start<40:
c.message(alice,chat,'telemetry-backend-offline')
n+=1
rss,queue,capacity=collector_sample()
memory.append(rss);queues.append(queue)
x=val(FAILED)
if x is not None:failures.append(x)
time.sleep(.3)
verify_ready(c,bob,att,raw,seconds=180)
check(n>0,'no successful writes during backend outage')
check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics')
rss,queue,capacity=collector_sample()
memory.append(rss);queues.append(queue)
print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True)
after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after)
check(max(memory)<memory_limit,f'Collector RSS {max(memory):g} bytes exceeds/equal to container limit {memory_limit} bytes')
check(max(queues)<=capacity,'collector queue exceeds configured cap')
result={'successful_messages_during_tempo_outage':n,'trace_id':tid,'max_sampled_collector_rss_bytes':max(memory),'collector_memory_limit_bytes':memory_limit,'rss_sample_count':len(memory),'max_export_queue':max(queues),'export_queue_capacity':capacity,'failed_span_counter_delta':max(failures)-baseline,'restart_state':after}
finally:compose('up','-d','tempo')
ready()
# Confirm a fresh post-recovery trace can be retrieved.
r=c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
tid2=r['headers']['X-Trace-Id']
wait_for('post-recovery trace',lambda:trace_spans(tid2),bool,seconds=90)
result['recovered_trace_id']=tid2
print('PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered')
return result
def metrics_snapshot(c):
alice,bob,_,chat=c.bootstrap()
att,raw,_,_=upload(c,alice,chat)
verify_ready(c,bob,att,raw,seconds=150)
until=time.monotonic()+12
while time.monotonic()<until:
c.message(alice,chat,'synthetic-metric-check')
time.sleep(5)
# Read the real provisioned Grafana dashboard/datasource list, without
# printing credentials or copying the response into the evidence.
secret=compose('exec','-T','grafana','printenv','GF_SECURITY_ADMIN_PASSWORD')
auth=base64.b64encode(('admin:'+secret).encode()).decode()
def grafana_json(path):
r=urllib.request.Request(GRAFANA+path,headers={'Authorization':'Basic '+auth})
try:
with urllib.request.urlopen(r,timeout=8) as f:return json.load(f)
except (urllib.error.URLError,TimeoutError,ValueError):
raise Failure('Grafana authenticated provisioning check failed; verify admin password/volume') from None
dashboard=grafana_json('/api/dashboards/uid/lab6-messenger')['dashboard']
check(len(dashboard['panels'])>=15,'provisioned dashboard incomplete')
check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing')
queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR,
'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count',
'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting',
'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'}
output={}
for key,expr in queries.items():
rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2)
if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load')
output[key]={'query':expr,'result':rows}
forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'}
for item in output.values():
check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels')
print('PASS metrics snapshot: all expected series present, no forbidden ID labels')
return output
def history(c):
alice,bob,_,chat=c.bootstrap()
att,raw,tid,rid=upload(c,alice,chat)
verify_ready(c,bob,att,raw,seconds=150)
wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90)
wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
time.sleep(6)
historic_time=time.time()
expr='sum(lab_http_requests_total{route=~"/api/.*"})'
def historic():
rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result']
return float(rows[0]['value'][1]) if rows else None
before=historic();check(before is not None and before>0,'no historic metric')
compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180)
ready()
wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100)
wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100)
after=historic();check(after==before,'historic metric changed or disappeared')
print('PASS history: saved metrics, logs and traces survive backend restart')
return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after}
def main():
p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args()
check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack')
ready();c=Client(timeout=12)
fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario]
result=fun(c)
file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
print('Saved',file.relative_to(ROOT))
if __name__=='__main__':run(main)