forked from mtusi2026/mtusi2026lab6
342 lines
19 KiB
Python
342 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab6 live evidence. Only synthetic test stack; no volume removal.
|
|
Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import math
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import urllib.error
|
|
from pathlib import Path
|
|
from client import Client, Failure, check, run
|
|
from pipeline_extended import compose, pg, scalar, wait_for
|
|
from pipeline_resilience import verify_ready
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
PROM='http://127.0.0.1:9090'
|
|
LOKI='http://127.0.0.1:3100'
|
|
TEMPO='http://127.0.0.1:3200'
|
|
GRAFANA='http://127.0.0.1:3000'
|
|
ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)'
|
|
LAG='max(lab_queue_oldest_age_seconds)'
|
|
DEPTH='max(lab_queue_depth)'
|
|
FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
|
|
# Explicit Prometheus readers can append unit suffixes to Collector self-metrics.
|
|
# Missing gauges are not zero: absent telemetry must fail the outage test.
|
|
QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})'
|
|
QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})'
|
|
RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
|
|
|
|
|
def get(url, params=None, *, as_json=True):
|
|
try:
|
|
with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r:
|
|
raw=r.read(32*1024*1024)
|
|
return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace')
|
|
except (urllib.error.URLError,TimeoutError,ValueError) as exc:
|
|
raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None
|
|
|
|
|
|
def prom(expr):
|
|
data=get(PROM+'/api/v1/query',{'query':expr})
|
|
check(data.get('status')=='success','PromQL query failed')
|
|
return data['data']['result']
|
|
|
|
|
|
def val(expr):
|
|
rows=prom(expr)
|
|
return float(rows[0]['value'][1]) if rows else None
|
|
|
|
|
|
def alert_state(name):
|
|
rules=get(PROM+'/api/v1/rules')['data']['groups']
|
|
for group in rules:
|
|
for rule in group['rules']:
|
|
if rule['name']==name:
|
|
check(rule.get('health')=='ok','alert rule is not healthy: '+name)
|
|
return rule['state']
|
|
raise Failure('alert rule missing: '+name)
|
|
|
|
|
|
def wait_alert(name,state,seconds=100):
|
|
return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2)
|
|
|
|
|
|
def ready():
|
|
for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'):
|
|
wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2)
|
|
wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100)
|
|
wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40)
|
|
wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40)
|
|
|
|
|
|
def trace_spans(tid):
|
|
raw=get(TEMPO+'/api/traces/'+tid)
|
|
spans=[]
|
|
def walk(obj):
|
|
if isinstance(obj,dict):
|
|
if {'spanId','traceId','name'}<=set(obj): spans.append(obj)
|
|
else:
|
|
for v in obj.values():walk(v)
|
|
elif isinstance(obj,list):
|
|
for v in obj:walk(v)
|
|
walk(raw)
|
|
return spans
|
|
|
|
|
|
def logs(tid,service,request_id=None):
|
|
expr='{service_name="%s"} | trace_id="%s"' % (service,tid)
|
|
if request_id: expr+=' | request_id="%s"'%request_id
|
|
data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100})
|
|
return data['data']['result']
|
|
|
|
|
|
def upload(c,alice,chat):
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
tid=r['headers'].get('X-Trace-Id','')
|
|
check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement')
|
|
return r['json'],raw,tid,r['headers']['X-Request-Id']
|
|
|
|
|
|
def span_attrs(s):
|
|
attrs={}
|
|
for a in s.get('attributes',[]):
|
|
value=a.get('value',{})
|
|
attrs[a['key']]=next(iter(value.values()),None)
|
|
return attrs
|
|
|
|
|
|
def chain_ok(spans):
|
|
names={s['name'] for s in spans}
|
|
required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'}
|
|
if not required<=names:return False
|
|
producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'}
|
|
consumers=[s for s in spans if s['name']=='rabbitmq.consume']
|
|
outcomes={span_attrs(s).get('job.outcome') for s in consumers}
|
|
return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers)
|
|
|
|
|
|
def trace_retry(c):
|
|
alice,bob,_,chat=c.bootstrap()
|
|
compose('stop','worker')
|
|
try:
|
|
att,raw,tid,rid=upload(c,alice,chat)
|
|
# Override only worker, never a public endpoint; reset in finally.
|
|
env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'}
|
|
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180)
|
|
verify_ready(c,bob,att,raw,seconds=180)
|
|
spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3)
|
|
wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
|
|
def worker_events(streams):
|
|
return {v[1] for row in streams for v in row.get('values',[])}
|
|
wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90)
|
|
# Keep only allowlisted names/IDs, not arbitrary span values or log payloads.
|
|
summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans]
|
|
result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True}
|
|
print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid)
|
|
return result
|
|
finally:
|
|
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180)
|
|
|
|
|
|
def queue_lag(c):
|
|
alice,bob,_,chat=c.bootstrap()
|
|
wait_alert('LabQueueLag','inactive')
|
|
compose('stop','worker')
|
|
try:
|
|
att,raw,tid,rid=upload(c,alice,chat)
|
|
t=time.monotonic()
|
|
wait_alert('LabQueueLag','firing')
|
|
lag=val(LAG);depth=val(DEPTH)
|
|
check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics')
|
|
detected=time.monotonic()-t
|
|
print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True)
|
|
compose('up','-d','worker')
|
|
verify_ready(c,bob,att,raw,seconds=180)
|
|
wait_alert('LabQueueLag','inactive',seconds=120)
|
|
wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2)
|
|
print('PASS queue-lag: firing -> ready -> resolved')
|
|
return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)}
|
|
finally:compose('up','-d','worker')
|
|
|
|
|
|
def db_error(c):
|
|
alice,_,_,_=c.bootstrap()
|
|
wait_alert('LabAPIHighErrorRate','inactive')
|
|
user=compose('exec','-T','postgres','printenv','POSTGRES_USER')
|
|
db=compose('exec','-T','postgres','printenv','POSTGRES_DB')
|
|
# Bounded, isolated table lock. Health SELECT 1 works, authenticated business
|
|
# read times out at statement_timeout. Never stop DB (LB would hide API 5xx).
|
|
cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;']
|
|
lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
try:
|
|
wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15)
|
|
start=time.monotonic(); traces=[]; codes=[]
|
|
while time.monotonic()-start < 43:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
|
def get_user(_):
|
|
r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User')
|
|
return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id')
|
|
for code,tid,rid in pool.map(get_user,range(3)):
|
|
codes.append(code)
|
|
if code==503 and tid:traces.append((tid,rid))
|
|
if traces and alert_state('LabAPIHighErrorRate')=='firing':break
|
|
check(bool(traces),'no real HTTP 503 from DB lock')
|
|
wait_alert('LabAPIHighErrorRate','firing',seconds=20)
|
|
ratio=val(ERR)
|
|
check(ratio is not None and ratio>0.05,'error ratio below threshold')
|
|
tid,rid=traces[0]
|
|
spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2)
|
|
wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60)
|
|
print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True)
|
|
finally:
|
|
# SQL transaction has server-side 60s bound; wait for normal COMMIT.
|
|
try:lock.wait(timeout=75)
|
|
except subprocess.TimeoutExpired:
|
|
lock.terminate();lock.wait(timeout=10)
|
|
# Keep successful business traffic in the evaluation window to prove recovery.
|
|
t=time.monotonic()
|
|
while time.monotonic()-t<65:
|
|
c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
|
|
if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break
|
|
time.sleep(1)
|
|
wait_alert('LabAPIHighErrorRate','inactive',seconds=40)
|
|
print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved')
|
|
return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]}
|
|
|
|
|
|
def collector_sample():
|
|
check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down')
|
|
rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY)
|
|
check(rss is not None and math.isfinite(rss) and rss>0,
|
|
'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus')
|
|
check(queue is not None and math.isfinite(queue) and queue>=0,
|
|
'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics')
|
|
check(capacity is not None and math.isfinite(capacity) and capacity==64,
|
|
'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity))
|
|
return rss,queue,capacity
|
|
|
|
|
|
def backend_down(c):
|
|
alice,bob,_,chat=c.bootstrap()
|
|
baseline=val(FAILED) or 0
|
|
cid=compose('ps','-q','otel-collector')
|
|
before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
|
|
memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip())
|
|
check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit))
|
|
rss,queue,capacity=collector_sample()
|
|
print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True)
|
|
compose('stop','tempo')
|
|
try:
|
|
start=time.monotonic();n=0;memory=[];queues=[];failures=[]
|
|
att,raw,tid,_=upload(c,alice,chat)
|
|
while time.monotonic()-start<40:
|
|
c.message(alice,chat,'telemetry-backend-offline')
|
|
n+=1
|
|
rss,queue,capacity=collector_sample()
|
|
memory.append(rss);queues.append(queue)
|
|
x=val(FAILED)
|
|
if x is not None:failures.append(x)
|
|
time.sleep(.3)
|
|
verify_ready(c,bob,att,raw,seconds=180)
|
|
check(n>0,'no successful writes during backend outage')
|
|
check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics')
|
|
rss,queue,capacity=collector_sample()
|
|
memory.append(rss);queues.append(queue)
|
|
print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True)
|
|
after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
|
|
check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after)
|
|
check(max(memory)<memory_limit,f'Collector RSS {max(memory):g} bytes exceeds/equal to container limit {memory_limit} bytes')
|
|
check(max(queues)<=capacity,'collector queue exceeds configured cap')
|
|
result={'successful_messages_during_tempo_outage':n,'trace_id':tid,'max_sampled_collector_rss_bytes':max(memory),'collector_memory_limit_bytes':memory_limit,'rss_sample_count':len(memory),'max_export_queue':max(queues),'export_queue_capacity':capacity,'failed_span_counter_delta':max(failures)-baseline,'restart_state':after}
|
|
finally:compose('up','-d','tempo')
|
|
ready()
|
|
# Confirm a fresh post-recovery trace can be retrieved.
|
|
r=c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
|
|
tid2=r['headers']['X-Trace-Id']
|
|
wait_for('post-recovery trace',lambda:trace_spans(tid2),bool,seconds=90)
|
|
result['recovered_trace_id']=tid2
|
|
print('PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered')
|
|
return result
|
|
|
|
|
|
def metrics_snapshot(c):
|
|
alice,bob,_,chat=c.bootstrap()
|
|
att,raw,_,_=upload(c,alice,chat)
|
|
verify_ready(c,bob,att,raw,seconds=150)
|
|
until=time.monotonic()+12
|
|
while time.monotonic()<until:
|
|
c.message(alice,chat,'synthetic-metric-check')
|
|
time.sleep(5)
|
|
# Read the real provisioned Grafana dashboard/datasource list, without
|
|
# printing credentials or copying the response into the evidence.
|
|
secret=compose('exec','-T','grafana','printenv','GF_SECURITY_ADMIN_PASSWORD')
|
|
auth=base64.b64encode(('admin:'+secret).encode()).decode()
|
|
def grafana_json(path):
|
|
r=urllib.request.Request(GRAFANA+path,headers={'Authorization':'Basic '+auth})
|
|
try:
|
|
with urllib.request.urlopen(r,timeout=8) as f:return json.load(f)
|
|
except (urllib.error.URLError,TimeoutError,ValueError):
|
|
raise Failure('Grafana authenticated provisioning check failed; verify admin password/volume') from None
|
|
dashboard=grafana_json('/api/dashboards/uid/lab6-messenger')['dashboard']
|
|
check(len(dashboard['panels'])>=15,'provisioned dashboard incomplete')
|
|
check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing')
|
|
queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR,
|
|
'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
|
'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
|
'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
|
'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count',
|
|
'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting',
|
|
'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'}
|
|
output={}
|
|
for key,expr in queries.items():
|
|
rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2)
|
|
if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load')
|
|
output[key]={'query':expr,'result':rows}
|
|
forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'}
|
|
for item in output.values():
|
|
check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels')
|
|
print('PASS metrics snapshot: all expected series present, no forbidden ID labels')
|
|
return output
|
|
|
|
|
|
def history(c):
|
|
alice,bob,_,chat=c.bootstrap()
|
|
att,raw,tid,rid=upload(c,alice,chat)
|
|
verify_ready(c,bob,att,raw,seconds=150)
|
|
wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90)
|
|
wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
|
|
time.sleep(6)
|
|
historic_time=time.time()
|
|
expr='sum(lab_http_requests_total{route=~"/api/.*"})'
|
|
def historic():
|
|
rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result']
|
|
return float(rows[0]['value'][1]) if rows else None
|
|
before=historic();check(before is not None and before>0,'no historic metric')
|
|
compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180)
|
|
ready()
|
|
wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100)
|
|
wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100)
|
|
after=historic();check(after==before,'historic metric changed or disappeared')
|
|
print('PASS history: saved metrics, logs and traces survive backend restart')
|
|
return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after}
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args()
|
|
check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack')
|
|
ready();c=Client(timeout=12)
|
|
fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario]
|
|
result=fun(c)
|
|
file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
|
|
print('Saved',file.relative_to(ROOT))
|
|
|
|
if __name__=='__main__':run(main)
|