#!/usr/bin/env python3 """Lab6 live evidence. Only synthetic test stack; no volume removal. Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads. """ import argparse import base64 import math import concurrent.futures import json import os import subprocess import time import urllib.parse import urllib.request import urllib.error from pathlib import Path from client import Client, Failure, check, run from pipeline_extended import compose, pg, scalar, wait_for from pipeline_resilience import verify_ready ROOT=Path(__file__).resolve().parents[1] PROM='http://127.0.0.1:9090' LOKI='http://127.0.0.1:3100' TEMPO='http://127.0.0.1:3200' GRAFANA='http://127.0.0.1:3000' ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)' LAG='max(lab_queue_oldest_age_seconds)' DEPTH='max(lab_queue_depth)' FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)' # Explicit Prometheus readers can append unit suffixes to Collector self-metrics. # Missing gauges are not zero: absent telemetry must fail the outage test. QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})' QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})' RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})' def get(url, params=None, *, as_json=True): try: with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r: raw=r.read(32*1024*1024) return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace') except (urllib.error.URLError,TimeoutError,ValueError) as exc: raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None def prom(expr): data=get(PROM+'/api/v1/query',{'query':expr}) check(data.get('status')=='success','PromQL query failed') return data['data']['result'] def val(expr): rows=prom(expr) return float(rows[0]['value'][1]) if rows else None def alert_state(name): rules=get(PROM+'/api/v1/rules')['data']['groups'] for group in rules: for rule in group['rules']: if rule['name']==name: check(rule.get('health')=='ok','alert rule is not healthy: '+name) return rule['state'] raise Failure('alert rule missing: '+name) def wait_alert(name,state,seconds=100): return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2) def ready(): for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'): wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2) wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100) wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40) wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40) def trace_spans(tid): raw=get(TEMPO+'/api/traces/'+tid) spans=[] def walk(obj): if isinstance(obj,dict): if {'spanId','traceId','name'}<=set(obj): spans.append(obj) else: for v in obj.values():walk(v) elif isinstance(obj,list): for v in obj:walk(v) walk(raw) return spans def logs(tid,service,request_id=None): expr='{service_name="%s"} | trace_id="%s"' % (service,tid) if request_id: expr+=' | request_id="%s"'%request_id data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100}) return data['data']['result'] def upload(c,alice,chat): raw=(ROOT/'tests/fixtures/sample.png').read_bytes() r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment') tid=r['headers'].get('X-Trace-Id','') check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement') return r['json'],raw,tid,r['headers']['X-Request-Id'] def span_attrs(s): attrs={} for a in s.get('attributes',[]): value=a.get('value',{}) attrs[a['key']]=next(iter(value.values()),None) return attrs def chain_ok(spans): names={s['name'] for s in spans} required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'} if not required<=names:return False producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'} consumers=[s for s in spans if s['name']=='rabbitmq.consume'] outcomes={span_attrs(s).get('job.outcome') for s in consumers} return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers) def trace_retry(c): alice,bob,_,chat=c.bootstrap() compose('stop','worker') try: att,raw,tid,rid=upload(c,alice,chat) # Override only worker, never a public endpoint; reset in finally. env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'} subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180) verify_ready(c,bob,att,raw,seconds=180) spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3) wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90) def worker_events(streams): return {v[1] for row in streams for v in row.get('values',[])} wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90) # Keep only allowlisted names/IDs, not arbitrary span values or log payloads. summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans] result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True} print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid) return result finally: subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180) def queue_lag(c): alice,bob,_,chat=c.bootstrap() wait_alert('LabQueueLag','inactive') compose('stop','worker') try: att,raw,tid,rid=upload(c,alice,chat) t=time.monotonic() wait_alert('LabQueueLag','firing') lag=val(LAG);depth=val(DEPTH) check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics') detected=time.monotonic()-t print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True) compose('up','-d','worker') verify_ready(c,bob,att,raw,seconds=180) wait_alert('LabQueueLag','inactive',seconds=120) wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2) print('PASS queue-lag: firing -> ready -> resolved') return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)} finally:compose('up','-d','worker') def db_error(c): alice,_,_,_=c.bootstrap() wait_alert('LabAPIHighErrorRate','inactive') user=compose('exec','-T','postgres','printenv','POSTGRES_USER') db=compose('exec','-T','postgres','printenv','POSTGRES_DB') # Bounded, isolated table lock. Health SELECT 1 works, authenticated business # read times out at statement_timeout. Never stop DB (LB would hide API 5xx). cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;'] lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) try: wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15) start=time.monotonic(); traces=[]; codes=[] while time.monotonic()-start < 43: with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: def get_user(_): r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User') return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id') for code,tid,rid in pool.map(get_user,range(3)): codes.append(code) if code==503 and tid:traces.append((tid,rid)) if traces and alert_state('LabAPIHighErrorRate')=='firing':break check(bool(traces),'no real HTTP 503 from DB lock') wait_alert('LabAPIHighErrorRate','firing',seconds=20) ratio=val(ERR) check(ratio is not None and ratio>0.05,'error ratio below threshold') tid,rid=traces[0] spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2) wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60) print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True) finally: # SQL transaction has server-side 60s bound; wait for normal COMMIT. try:lock.wait(timeout=75) except subprocess.TimeoutExpired: lock.terminate();lock.wait(timeout=10) # Keep successful business traffic in the evaluation window to prove recovery. t=time.monotonic() while time.monotonic()-t<65: c.call('GET','/api/v1/users/me',200,actor=alice,schema='User') if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break time.sleep(1) wait_alert('LabAPIHighErrorRate','inactive',seconds=40) print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved') return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]} def collector_sample(): check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down') rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY) check(rss is not None and math.isfinite(rss) and rss>0, 'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus') check(queue is not None and math.isfinite(queue) and queue>=0, 'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics') check(capacity is not None and math.isfinite(capacity) and capacity==64, 'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity)) return rss,queue,capacity def backend_down(c): alice,bob,_,chat=c.bootstrap() baseline=val(FAILED) or 0 cid=compose('ps','-q','otel-collector') before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip() memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip()) check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit)) rss,queue,capacity=collector_sample() print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True) compose('stop','tempo') try: start=time.monotonic();n=0;memory=[];queues=[];failures=[] att,raw,tid,_=upload(c,alice,chat) while time.monotonic()-start<40: c.message(alice,chat,'telemetry-backend-offline') n+=1 rss,queue,capacity=collector_sample() memory.append(rss);queues.append(queue) x=val(FAILED) if x is not None:failures.append(x) time.sleep(.3) verify_ready(c,bob,att,raw,seconds=180) check(n>0,'no successful writes during backend outage') check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics') rss,queue,capacity=collector_sample() memory.append(rss);queues.append(queue) print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True) after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip() check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after) check(max(memory)=15,'provisioned dashboard incomplete') check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing') queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR, 'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))', 'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))', 'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))', 'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count', 'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting', 'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'} output={} for key,expr in queries.items(): rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2) if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load') output[key]={'query':expr,'result':rows} forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'} for item in output.values(): check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels') print('PASS metrics snapshot: all expected series present, no forbidden ID labels') return output def history(c): alice,bob,_,chat=c.bootstrap() att,raw,tid,rid=upload(c,alice,chat) verify_ready(c,bob,att,raw,seconds=150) wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90) wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90) time.sleep(6) historic_time=time.time() expr='sum(lab_http_requests_total{route=~"/api/.*"})' def historic(): rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result'] return float(rows[0]['value'][1]) if rows else None before=historic();check(before is not None and before>0,'no historic metric') compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180) ready() wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100) wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100) after=historic();check(after==before,'historic metric changed or disappeared') print('PASS history: saved metrics, logs and traces survive backend restart') return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after} def main(): p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args() check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack') ready();c=Client(timeout=12) fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario] result=fun(c) file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8') print('Saved',file.relative_to(ROOT)) if __name__=='__main__':run(main)