forked from mtusi2026/mtusi2026lab6
208 lines
12 KiB
Python
208 lines
12 KiB
Python
"""Short real-backend check; optional bounded incidents for the grade-4 evidence."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
EVIDENCE=ROOT/'evidence'
|
|
|
|
|
|
def get(url):
|
|
request=urllib.request.Request(url,headers={'Accept':'application/json'})
|
|
with urllib.request.urlopen(request,timeout=5) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def wait(fn, label, seconds=35):
|
|
deadline=time.monotonic()+seconds
|
|
error=None
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
result=fn()
|
|
if result:return result
|
|
except (OSError,ValueError,Failure) as exc:error=type(exc).__name__
|
|
time.sleep(.5)
|
|
raise AssertionError('Timeout '+label+' '+str(error))
|
|
|
|
|
|
def prom(expression):
|
|
return get('http://localhost:'+os.environ.get('PROMETHEUS_PORT','9090')+'/api/v1/query?'+urllib.parse.urlencode({'query':expression}))['data']['result']
|
|
|
|
|
|
def value(expression):
|
|
result=prom(expression)
|
|
return float(result[0]['value'][1]) if result else 0
|
|
|
|
|
|
def logs(trace_id=None, event=None):
|
|
query='{service_name=~"messenger-.*"} | json'
|
|
if trace_id:query+=' | trace_id="'+trace_id+'"'
|
|
if event:query+=' | event="'+event+'"'
|
|
data=get('http://localhost:'+os.environ.get('LOKI_PORT','3100')+'/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':query,'limit':200,'start':str(int((time.time()-600)*1e9))}))
|
|
return [json.loads(row[1]) for stream in data['data']['result'] for row in stream['values']]
|
|
|
|
|
|
def trace_data(trace_id):
|
|
return get('http://localhost:'+os.environ.get('TEMPO_PORT','3200')+'/api/traces/'+trace_id)
|
|
|
|
|
|
def spans(data):
|
|
if isinstance(data,dict):
|
|
if 'spanId' in data and 'name' in data:yield data
|
|
for child in data.values():yield from spans(child)
|
|
elif isinstance(data,list):
|
|
for child in data:yield from spans(child)
|
|
|
|
|
|
def compose(*parts, input=None):
|
|
result=subprocess.run(['docker','compose',*parts],cwd=ROOT,input=input,text=True,capture_output=True,timeout=60)
|
|
check(result.returncode==0,'Compose failed: '+' '.join(parts[:3])+' '+result.stderr[-400:])
|
|
return result.stdout.strip()
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--incidents',action='store_true')
|
|
p.add_argument('--from-db',action='store_true',help='Continue previously verified worker incident')
|
|
args=arguments(__doc__,configure)
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
results=[]
|
|
def record(case,**fields):
|
|
row=dict(case=case,result='PASS',**fields);results.append(row)
|
|
print(json.dumps(row,ensure_ascii=False),flush=True)
|
|
c=Client(args.base_url or 'http://localhost:8086',timeout=12)
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
trace_id=uuid.uuid4().hex
|
|
content=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,
|
|
actor=alice,raw=content,headers={'Content-Type':'image/png','traceparent':'00-'+trace_id+'-1234567890abcdef-01'},schema='Attachment')
|
|
check(response['headers']['X-Trace-Id']==trace_id,'incoming W3C context not preserved')
|
|
att=c.wait_ready(alice,response['json'])
|
|
def complete_trace():
|
|
d=trace_data(trace_id)
|
|
names={span['name'] for span in spans(d)}
|
|
return d if {'queue.publish','queue.consume','image.transform'} <= names else None
|
|
data=wait(complete_trace,'trace pipeline')
|
|
entries=list(spans(data));names={span['name'] for span in entries}
|
|
check({'queue.publish','queue.consume','image.transform'}.issubset(names),'producer/consumer/transform missing')
|
|
check(any(name.startswith('s3.') for name in names) and any(name.startswith('db.') for name in names),'storage spans missing')
|
|
producer=next(span for span in entries if span['name']=='queue.publish')
|
|
consumer=next(span for span in entries if span['name']=='queue.consume')
|
|
check(consumer['parentSpanId']==producer['spanId'],'consumer must be child of producer')
|
|
check(any(span['spanId']==producer['parentSpanId'] and '/attachments' in span['name'] for span in entries),'producer must be child of upload')
|
|
def correlated():
|
|
rows=logs(trace_id)
|
|
return rows if {'http.completed','job.completed'}.issubset({row['event'] for row in rows}) else None
|
|
linked=wait(correlated,'correlated logs')
|
|
for row in linked:check(row['trace_id']==trace_id and len(row['span_id'])==16,'log correlation')
|
|
check(not any(alice['token'] in json.dumps(d) or alice['registration']['password'] in json.dumps(d) for d in [data,linked]),'secret in signals')
|
|
(EVIDENCE/'lab6-trace.json').write_text(json.dumps(data,ensure_ascii=False,indent=2))
|
|
(EVIDENCE/'lab6-correlated-logs.json').write_text(json.dumps(linked,ensure_ascii=False,indent=2))
|
|
record('upload_to_worker_trace_and_logs',trace_id=trace_id,spans=len(entries))
|
|
required=['http_server_requests_total','http_server_duration_seconds_bucket','worker_jobs_total',
|
|
'worker_job_duration_seconds_bucket','attachment_ready_duration_seconds_bucket','queue_depth',
|
|
'queue_oldest_age_seconds','db_pool_used','db_pool_waiting','worker_busy','worker_capacity',
|
|
'process_memory_rss_bytes','process_cpu_time_seconds_total']
|
|
series=wait(lambda: prom('count by(__name__) ({job="messenger"})'),'metrics')
|
|
found={row['metric']['__name__'] for row in series}
|
|
check(set(required)<=found,'metrics missing: '+str(set(required)-found))
|
|
wait(lambda:prom('sum(http_server_requests_total{route=~"/api/v1/.*"})'),'business route metrics')
|
|
labels=prom('http_server_requests_total')
|
|
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','trace_id','url'}
|
|
check(all(not forbidden.intersection(row['metric']) for row in labels),'high cardinality labels')
|
|
dashboard=get('http://localhost:'+os.environ.get('GRAFANA_PORT','3000')+'/api/dashboards/uid/messenger-lab6')
|
|
check(dashboard['meta']['provisioned'],'dashboard not provisioned')
|
|
for panel in dashboard['dashboard']['panels']:
|
|
if panel['datasource']['uid']=='prometheus':
|
|
for target in panel['targets']:prom(target['expr'])
|
|
record('metrics_labels_and_dashboard_queries',metrics=len(required),panels=len(dashboard['dashboard']['panels']))
|
|
if args.incidents:
|
|
worker_stopped=False;tempo_stopped=False;minio_paused=False;locker=None
|
|
try:
|
|
if not args.from_db:
|
|
compose('stop','-t','2','worker');worker_stopped=True
|
|
retry_trace=uuid.uuid4().hex
|
|
queued=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=retry.png',202,
|
|
actor=alice,raw=content,headers={'Content-Type':'image/png','traceparent':'00-'+retry_trace+'-1234567890abcdef-01'},schema='Attachment')['json']
|
|
c.message(alice,chat,'worker outage',attachments=[queued['id']])
|
|
wait(lambda:value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==1,'queue alert firing',35)
|
|
record('incident_worker_firing',queue_depth=value('max(queue_depth)'),oldest_seconds=value('max(queue_oldest_age_seconds)'))
|
|
compose('pause','minio');minio_paused=True
|
|
worker_id=compose('ps','-aq','worker')
|
|
subprocess.run(['docker','start',worker_id],check=True,stdout=subprocess.DEVNULL)
|
|
worker_stopped=False
|
|
wait(lambda:value('sum(worker_jobs_total{outcome="retry"})')>0,'transient retry',25)
|
|
compose('unpause','minio');minio_paused=False
|
|
wait(lambda:c.call('GET','/health/ready')['status']==200,'API restored')
|
|
c.wait_ready(alice,queued)
|
|
def retry_chain():
|
|
d=trace_data(retry_trace)
|
|
return d if sum(span['name']=='queue.consume' for span in spans(d))>=2 else None
|
|
retry_data=wait(retry_chain,'retry chain')
|
|
(EVIDENCE/'lab6-retry-trace.json').write_text(json.dumps(retry_data,indent=2))
|
|
record('retry_keeps_upload_trace',trace_id=retry_trace,consumer_spans=sum(span['name']=='queue.consume' for span in spans(retry_data)))
|
|
wait(lambda:value('max(queue_depth)')==0 and value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==0,'queue alert resolved')
|
|
record('incident_worker_resolved')
|
|
# Block INSERT while SELECT readiness remains available. This isolates
|
|
# DB latency from application CPU/HTTP handler latency.
|
|
api=compose('ps','-q','api').split()[0]
|
|
code='''import asyncio,asyncpg,sys
|
|
from src.core.settings import settings
|
|
async def main():
|
|
c=await asyncpg.connect(settings.database_url.get_secret_value())
|
|
await c.execute('BEGIN; LOCK TABLE messages IN SHARE MODE')
|
|
print('LOCKED',flush=True)
|
|
await asyncio.to_thread(sys.stdin.readline)
|
|
await c.execute('ROLLBACK');await c.close()
|
|
asyncio.run(main())'''
|
|
locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',code],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True)
|
|
check(locker.stdout.readline().strip()=='LOCKED','DB fault not active')
|
|
def failing(_):
|
|
return Client(c.base,timeout=12).call('POST','/api/v1/chats/'+chat['id']+'/messages',503,
|
|
actor=alice,body={'text':'bounded DB incident'},headers={'Idempotency-Key':uuid.uuid4().hex})
|
|
with ThreadPoolExecutor(max_workers=4) as pool:errors=list(pool.map(failing,range(12)))
|
|
wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==1,'error alert firing',20)
|
|
db_trace=wait(lambda: trace_data(errors[0]['headers']['X-Trace-Id']),'DB trace')
|
|
failures=[span for span in spans(db_trace) if span['name'].startswith('db.') and span.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR')]
|
|
check(failures,'DB error span missing')
|
|
db_logs=wait(lambda:logs(event='db.error'),'DB error logs')
|
|
(EVIDENCE/'lab6-db-incident-trace.json').write_text(json.dumps(db_trace,indent=2))
|
|
(EVIDENCE/'lab6-db-incident-logs.json').write_text(json.dumps(db_logs,indent=2))
|
|
record('incident_DB_firing',errors=len(errors),db_error_spans=len(failures),
|
|
diagnosis='db.INSERT timeout, matching db.error event; API readiness remains available')
|
|
locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10);locker=None
|
|
c.message(alice,chat,'DB recovered')
|
|
wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==0,'error alert resolved',40)
|
|
record('incident_DB_resolved')
|
|
compose('stop','-t','2','tempo');tempo_stopped=True
|
|
started=time.monotonic()
|
|
for _ in range(15):c.message(alice,chat,'telemetry outage')
|
|
other=c.attachment(alice,chat)[0];c.wait_ready(alice,other)
|
|
# Bounded retry is 10s; wait only for an observed failure, not a benchmark.
|
|
wait(lambda:value('sum(otelcol_exporter_send_failed_spans_total)')>0 or value('sum(otelcol_exporter_send_failed_spans)')>0,'export failure counter',20)
|
|
usage=json.loads(compose('exec','-T','api','python','-c','from pathlib import Path;import json;print(json.dumps({"bytes":int(Path("/sys/fs/cgroup/memory.current").read_text()),"limit":int(Path("/sys/fs/cgroup/memory.max").read_text())}))').splitlines()[0])
|
|
check(usage['bytes']<usage['limit'],'API memory exceeded cap')
|
|
record('telemetry_backend_outage_business_survives',messages=15,attachment_ready=True,api_memory=usage,
|
|
collector_rss_bytes=value('max(otelcol_process_memory_rss_bytes)'),
|
|
exporter_failed_spans=value('sum(otelcol_exporter_send_failed_spans_total)'),
|
|
max_exporter_queue=value('max(otelcol_exporter_queue_size)'))
|
|
compose('start','tempo');tempo_stopped=False
|
|
finally:
|
|
if locker:
|
|
locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10)
|
|
if minio_paused:compose('unpause','minio')
|
|
if worker_stopped:compose('start','worker')
|
|
if tempo_stopped:compose('start','tempo')
|
|
(EVIDENCE/'lab6-observability-summary.json').write_text(json.dumps({'results':results},ensure_ascii=False,indent=2))
|
|
|
|
|
|
if __name__=='__main__':run(main)
|