Files
mtusi2026lab6/tests/observability.py
T

208 lines
12 KiB
Python

"""Short real-backend check; optional bounded incidents for the grade-4 evidence."""
import json
import os
import subprocess
import time
import urllib.parse
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from client import Client, Failure, arguments, check, run
ROOT=Path(__file__).resolve().parents[1]
EVIDENCE=ROOT/'evidence'
def get(url):
request=urllib.request.Request(url,headers={'Accept':'application/json'})
with urllib.request.urlopen(request,timeout=5) as response:
return json.load(response)
def wait(fn, label, seconds=35):
deadline=time.monotonic()+seconds
error=None
while time.monotonic()<deadline:
try:
result=fn()
if result:return result
except (OSError,ValueError,Failure) as exc:error=type(exc).__name__
time.sleep(.5)
raise AssertionError('Timeout '+label+' '+str(error))
def prom(expression):
return get('http://localhost:'+os.environ.get('PROMETHEUS_PORT','9090')+'/api/v1/query?'+urllib.parse.urlencode({'query':expression}))['data']['result']
def value(expression):
result=prom(expression)
return float(result[0]['value'][1]) if result else 0
def logs(trace_id=None, event=None):
query='{service_name=~"messenger-.*"} | json'
if trace_id:query+=' | trace_id="'+trace_id+'"'
if event:query+=' | event="'+event+'"'
data=get('http://localhost:'+os.environ.get('LOKI_PORT','3100')+'/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':query,'limit':200,'start':str(int((time.time()-600)*1e9))}))
return [json.loads(row[1]) for stream in data['data']['result'] for row in stream['values']]
def trace_data(trace_id):
return get('http://localhost:'+os.environ.get('TEMPO_PORT','3200')+'/api/traces/'+trace_id)
def spans(data):
if isinstance(data,dict):
if 'spanId' in data and 'name' in data:yield data
for child in data.values():yield from spans(child)
elif isinstance(data,list):
for child in data:yield from spans(child)
def compose(*parts, input=None):
result=subprocess.run(['docker','compose',*parts],cwd=ROOT,input=input,text=True,capture_output=True,timeout=60)
check(result.returncode==0,'Compose failed: '+' '.join(parts[:3])+' '+result.stderr[-400:])
return result.stdout.strip()
def main():
def configure(p):
p.add_argument('--incidents',action='store_true')
p.add_argument('--from-db',action='store_true',help='Continue previously verified worker incident')
args=arguments(__doc__,configure)
EVIDENCE.mkdir(exist_ok=True)
results=[]
def record(case,**fields):
row=dict(case=case,result='PASS',**fields);results.append(row)
print(json.dumps(row,ensure_ascii=False),flush=True)
c=Client(args.base_url or 'http://localhost:8086',timeout=12)
alice,bob,eve,chat=c.bootstrap()
trace_id=uuid.uuid4().hex
content=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,
actor=alice,raw=content,headers={'Content-Type':'image/png','traceparent':'00-'+trace_id+'-1234567890abcdef-01'},schema='Attachment')
check(response['headers']['X-Trace-Id']==trace_id,'incoming W3C context not preserved')
att=c.wait_ready(alice,response['json'])
def complete_trace():
d=trace_data(trace_id)
names={span['name'] for span in spans(d)}
return d if {'queue.publish','queue.consume','image.transform'} <= names else None
data=wait(complete_trace,'trace pipeline')
entries=list(spans(data));names={span['name'] for span in entries}
check({'queue.publish','queue.consume','image.transform'}.issubset(names),'producer/consumer/transform missing')
check(any(name.startswith('s3.') for name in names) and any(name.startswith('db.') for name in names),'storage spans missing')
producer=next(span for span in entries if span['name']=='queue.publish')
consumer=next(span for span in entries if span['name']=='queue.consume')
check(consumer['parentSpanId']==producer['spanId'],'consumer must be child of producer')
check(any(span['spanId']==producer['parentSpanId'] and '/attachments' in span['name'] for span in entries),'producer must be child of upload')
def correlated():
rows=logs(trace_id)
return rows if {'http.completed','job.completed'}.issubset({row['event'] for row in rows}) else None
linked=wait(correlated,'correlated logs')
for row in linked:check(row['trace_id']==trace_id and len(row['span_id'])==16,'log correlation')
check(not any(alice['token'] in json.dumps(d) or alice['registration']['password'] in json.dumps(d) for d in [data,linked]),'secret in signals')
(EVIDENCE/'lab6-trace.json').write_text(json.dumps(data,ensure_ascii=False,indent=2))
(EVIDENCE/'lab6-correlated-logs.json').write_text(json.dumps(linked,ensure_ascii=False,indent=2))
record('upload_to_worker_trace_and_logs',trace_id=trace_id,spans=len(entries))
required=['http_server_requests_total','http_server_duration_seconds_bucket','worker_jobs_total',
'worker_job_duration_seconds_bucket','attachment_ready_duration_seconds_bucket','queue_depth',
'queue_oldest_age_seconds','db_pool_used','db_pool_waiting','worker_busy','worker_capacity',
'process_memory_rss_bytes','process_cpu_time_seconds_total']
series=wait(lambda: prom('count by(__name__) ({job="messenger"})'),'metrics')
found={row['metric']['__name__'] for row in series}
check(set(required)<=found,'metrics missing: '+str(set(required)-found))
wait(lambda:prom('sum(http_server_requests_total{route=~"/api/v1/.*"})'),'business route metrics')
labels=prom('http_server_requests_total')
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','trace_id','url'}
check(all(not forbidden.intersection(row['metric']) for row in labels),'high cardinality labels')
dashboard=get('http://localhost:'+os.environ.get('GRAFANA_PORT','3000')+'/api/dashboards/uid/messenger-lab6')
check(dashboard['meta']['provisioned'],'dashboard not provisioned')
for panel in dashboard['dashboard']['panels']:
if panel['datasource']['uid']=='prometheus':
for target in panel['targets']:prom(target['expr'])
record('metrics_labels_and_dashboard_queries',metrics=len(required),panels=len(dashboard['dashboard']['panels']))
if args.incidents:
worker_stopped=False;tempo_stopped=False;minio_paused=False;locker=None
try:
if not args.from_db:
compose('stop','-t','2','worker');worker_stopped=True
retry_trace=uuid.uuid4().hex
queued=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=retry.png',202,
actor=alice,raw=content,headers={'Content-Type':'image/png','traceparent':'00-'+retry_trace+'-1234567890abcdef-01'},schema='Attachment')['json']
c.message(alice,chat,'worker outage',attachments=[queued['id']])
wait(lambda:value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==1,'queue alert firing',35)
record('incident_worker_firing',queue_depth=value('max(queue_depth)'),oldest_seconds=value('max(queue_oldest_age_seconds)'))
compose('pause','minio');minio_paused=True
worker_id=compose('ps','-aq','worker')
subprocess.run(['docker','start',worker_id],check=True,stdout=subprocess.DEVNULL)
worker_stopped=False
wait(lambda:value('sum(worker_jobs_total{outcome="retry"})')>0,'transient retry',25)
compose('unpause','minio');minio_paused=False
wait(lambda:c.call('GET','/health/ready')['status']==200,'API restored')
c.wait_ready(alice,queued)
def retry_chain():
d=trace_data(retry_trace)
return d if sum(span['name']=='queue.consume' for span in spans(d))>=2 else None
retry_data=wait(retry_chain,'retry chain')
(EVIDENCE/'lab6-retry-trace.json').write_text(json.dumps(retry_data,indent=2))
record('retry_keeps_upload_trace',trace_id=retry_trace,consumer_spans=sum(span['name']=='queue.consume' for span in spans(retry_data)))
wait(lambda:value('max(queue_depth)')==0 and value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==0,'queue alert resolved')
record('incident_worker_resolved')
# Block INSERT while SELECT readiness remains available. This isolates
# DB latency from application CPU/HTTP handler latency.
api=compose('ps','-q','api').split()[0]
code='''import asyncio,asyncpg,sys
from src.core.settings import settings
async def main():
c=await asyncpg.connect(settings.database_url.get_secret_value())
await c.execute('BEGIN; LOCK TABLE messages IN SHARE MODE')
print('LOCKED',flush=True)
await asyncio.to_thread(sys.stdin.readline)
await c.execute('ROLLBACK');await c.close()
asyncio.run(main())'''
locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',code],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True)
check(locker.stdout.readline().strip()=='LOCKED','DB fault not active')
def failing(_):
return Client(c.base,timeout=12).call('POST','/api/v1/chats/'+chat['id']+'/messages',503,
actor=alice,body={'text':'bounded DB incident'},headers={'Idempotency-Key':uuid.uuid4().hex})
with ThreadPoolExecutor(max_workers=4) as pool:errors=list(pool.map(failing,range(12)))
wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==1,'error alert firing',20)
db_trace=wait(lambda: trace_data(errors[0]['headers']['X-Trace-Id']),'DB trace')
failures=[span for span in spans(db_trace) if span['name'].startswith('db.') and span.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR')]
check(failures,'DB error span missing')
db_logs=wait(lambda:logs(event='db.error'),'DB error logs')
(EVIDENCE/'lab6-db-incident-trace.json').write_text(json.dumps(db_trace,indent=2))
(EVIDENCE/'lab6-db-incident-logs.json').write_text(json.dumps(db_logs,indent=2))
record('incident_DB_firing',errors=len(errors),db_error_spans=len(failures),
diagnosis='db.INSERT timeout, matching db.error event; API readiness remains available')
locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10);locker=None
c.message(alice,chat,'DB recovered')
wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==0,'error alert resolved',40)
record('incident_DB_resolved')
compose('stop','-t','2','tempo');tempo_stopped=True
started=time.monotonic()
for _ in range(15):c.message(alice,chat,'telemetry outage')
other=c.attachment(alice,chat)[0];c.wait_ready(alice,other)
# Bounded retry is 10s; wait only for an observed failure, not a benchmark.
wait(lambda:value('sum(otelcol_exporter_send_failed_spans_total)')>0 or value('sum(otelcol_exporter_send_failed_spans)')>0,'export failure counter',20)
usage=json.loads(compose('exec','-T','api','python','-c','from pathlib import Path;import json;print(json.dumps({"bytes":int(Path("/sys/fs/cgroup/memory.current").read_text()),"limit":int(Path("/sys/fs/cgroup/memory.max").read_text())}))').splitlines()[0])
check(usage['bytes']<usage['limit'],'API memory exceeded cap')
record('telemetry_backend_outage_business_survives',messages=15,attachment_ready=True,api_memory=usage,
collector_rss_bytes=value('max(otelcol_process_memory_rss_bytes)'),
exporter_failed_spans=value('sum(otelcol_exporter_send_failed_spans_total)'),
max_exporter_queue=value('max(otelcol_exporter_queue_size)'))
compose('start','tempo');tempo_stopped=False
finally:
if locker:
locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10)
if minio_paused:compose('unpause','minio')
if worker_stopped:compose('start','worker')
if tempo_stopped:compose('start','tempo')
(EVIDENCE/'lab6-observability-summary.json').write_text(json.dumps({'results':results},ensure_ascii=False,indent=2))
if __name__=='__main__':run(main)