Files
mtusi2026lab6/student-tests/observability.py
T

258 lines
13 KiB
Python

"""Real OTLP evidence, controlled incidents and bounded telemetry-backend outage."""
import argparse
import base64
import concurrent.futures
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from urllib.parse import urlencode
import urllib.request
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure
BASE=os.environ.get('BASE_URL','http://127.0.0.1:18093')
PROM=os.environ.get('PROMETHEUS_URL','http://127.0.0.1:19096')
LOKI=os.environ.get('LOKI_URL','http://127.0.0.1:13106')
TEMPO=os.environ.get('TEMPO_URL','http://127.0.0.1:13206')
COLLECTOR=os.environ.get('COLLECTOR_URL','http://127.0.0.1:18896')
OPENER=urllib.request.build_opener(urllib.request.ProxyHandler({}))
def get(url):
req=urllib.request.Request(url,headers={'Accept':'application/json'})
with OPENER.open(req,timeout=10) as response:
return json.load(response)
def prom(query):
return get(PROM+'/api/v1/query?'+urlencode({'query':query}))['data']['result']
def scalar(query):
rows=prom(query)
return float(rows[0]['value'][1]) if rows else 0
def logs(query, start=None, limit=2000):
params={'query':query,'start':str(int((start or time.time()-600)*1e9)),
'end':str(time.time_ns()),'limit':str(limit),'direction':'forward'}
result=get(LOKI+'/loki/api/v1/query_range?'+urlencode(params))['data']['result']
return [json.loads(value) for stream in result for stamp,value in stream['values']]
def trace(identifier):
return get(TEMPO+'/api/traces/'+identifier)
def spans(data):
result=[]
for batch in data.get('batches',data.get('resourceSpans',[])):
service=next(a['value']['stringValue'] for a in batch['resource']['attributes'] if a['key']=='service.name')
for scope in batch.get('scopeSpans',[]):
for span in scope['spans']:
result.append(dict(span,service=service))
return result
def wait(function,timeout=60):
end=time.monotonic()+timeout
while time.monotonic()<end:
try:
result=function()
if result:return result
except (OSError,ValueError,Failure):
pass
time.sleep(.5)
raise AssertionError('Timed out: '+function.__name__)
def compose(*args):
return subprocess.check_output(['docker','compose',*args],text=True).strip()
def save(name,data):
(ROOT/'evidence'/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
def upload(c,actor,chat,tid):
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
actor=actor,raw=raw,headers={'Content-Type':'image/png',
'traceparent':'00-'+tid+'-0123456789abcdef-01'},schema='Attachment')
def proof():
wait(lambda:get(LOKI+'/loki/api/v1/labels'))
wait(lambda:get(TEMPO+'/api/search'))
proof_start=time.time()
c=Client(BASE);a,b,e,chat=c.bootstrap()
tid=uuid.uuid4().hex
marker='SECRET_MESSAGE_CANARY_'+uuid.uuid4().hex
c.message(a,chat,marker)
response=upload(c,a,chat,tid)
ready=c.wait_ready(b,response['json'])
def completed_trace():
data=trace(tid)
return data if any(s['name']=='attachment process' for s in spans(data)) else None
data=wait(completed_trace)
rows=spans(data)
upload_span=next(s for s in rows if s['kind']=='SPAN_KIND_SERVER')
publish=next(s for s in rows if s['kind']=='SPAN_KIND_PRODUCER')
consumer=next(s for s in rows if s['kind']=='SPAN_KIND_CONSUMER')
assert consumer['parentSpanId']==publish['spanId']
index={s['spanId']:s for s in rows}
parent=publish
while parent['spanId']!=upload_span['spanId']:
parent=index[parent['parentSpanId']]
assert any(s['service']=='messenger-worker' and s['name'].startswith('s3.') for s in rows)
assert any(s['service']=='messenger-worker' and s['name']=='db.transaction' for s in rows)
query='{service_name=~"messenger-.*"} | json | trace_id="'+tid+'"'
records=wait(lambda: logs(query) if len(logs(query))>=4 else None)
assert {r['event'] for r in records}>={'http_request','attachment_accepted','job_published','job_processed'}
assert any(r.get('request_id')==response['headers'].get('X-Request-Id') for r in records)
assert all(len(r['trace_id'])==32 and len(r['span_id'])==16 for r in records)
all_logs=logs('{service_name=~"messenger-.*"}',start=proof_start,limit=5000)
payload=json.dumps(all_logs)+json.dumps(data)
for value in [marker,a['token'],a['registration']['password'],a['registration']['username'],'private-canary.png']:
assert value not in payload, 'business/credential canary leaked'
metric_rows=prom('{service_name=~"messenger-.*"}')
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','job_id','trace_id','span_id'}
assert all(not forbidden.intersection(row['metric']) for row in metric_rows)
names={r['metric']['__name__'] for r in metric_rows}
assert {'http_server_duration_seconds_bucket','jobs_processed_total','queue_depth','queue_oldest_age_seconds','db_pool_used','worker_capacity'}<=names
save('trace-proof.json',{'trace_id':tid,'request_id':response['headers'].get('X-Request-Id'),
'status':ready['status'],'consumer_parent_matches_producer':True,'producer_descends_from_upload':True,
'api_worker_logs_correlated':True,'secret_canaries_absent':True,'metric_labels_allowlisted':True,
'trace':data,'logs':records})
# Retry is a real S3 outage after durable acceptance, not fabricated spans.
compose('stop','worker')
retry_tid=uuid.uuid4().hex
queued=upload(c,a,chat,retry_tid)['json']
paused=False
try:
compose('pause','s3');paused=True
compose('up','-d','--no-deps','worker')
def attempted():
raw=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
"SELECT attempt FROM attachment_jobs WHERE 1=1 AND attachment_id = '"+queued['id']+"';")
return int(raw)>=1
wait(attempted)
time.sleep(5)
finally:
if paused:compose('unpause','s3')
compose('up','-d','--no-deps','worker')
wait(lambda:c.call('GET','/health/ready',schema='Health'))
c.wait_ready(a,queued)
def retried():
data=trace(retry_tid)
return data if len([s for s in spans(data) if s['kind']=='SPAN_KIND_CONSUMER'])>=2 else None
retry_data=wait(retried)
retry_records=wait(lambda:logs('{service_name="messenger-worker"} | json | trace_id="'+retry_tid+'" | event="job_retry"'))
save('retry-trace.json',{'trace_id':retry_tid,'retry_observed':True,'trace':retry_data,'logs':retry_records})
print('PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.')
def alerts():
return get(PROM+'/api/v1/alerts')['data']['alerts']
def firing(name):
return [a for a in alerts() if a['labels']['alertname']==name and a['state']=='firing']
def incidents():
c=Client(BASE);a,b,e,chat=c.bootstrap();result={}
start=time.time()
compose('stop','worker')
try:
queued=[c.attachment(a,chat)[0] for _ in range(3)]
observed=wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')>=3)
fired=wait(lambda:firing('QueueLag'))
detected=time.time()
metrics={q:prom(q) for q in ['queue_depth','queue_oldest_age_seconds','worker_busy','worker_capacity']}
result['worker_stop']={'injected_at':start,'detected_at':detected,
'detection_seconds':detected-start,'firing':fired,'metrics':metrics,
'diagnosis':'Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.'}
finally:
compose('up','-d','--no-deps','worker')
for att in queued:c.wait_ready(a,att)
wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')==0)
wait(lambda:not firing('QueueLag'))
result['worker_stop']['resolved_at']=time.time()
# Keep readiness SELECT 1 available while real session SQL is blocked.
lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1',
'-c',"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;"],
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
time.sleep(1)
db_start=time.time();stop=False;errors=[];error_trace=uuid.uuid4().hex
def traffic():
while not stop:
r=c.call('GET','/api/v1/users/me',[200,503],actor=a,
headers={'traceparent':'00-'+error_trace+'-0123456789abcdef-01'})
if r['status']==503:errors.append(r['elapsed'])
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
futures=[pool.submit(traffic) for _ in range(4)]
try:
fired=wait(lambda:firing('APIHighErrorRate'),timeout=50)
detected=time.time()
result['db_lock']={'injected_at':db_start,'detected_at':detected,'detection_seconds':detected-db_start,
'firing':fired,'error_latencies_seconds':errors.copy(),'trace_id':error_trace}
lock.wait(timeout=45)
finally:
stop=True
for future in futures:future.result()
finally:
stop=True
if lock.poll() is None:
lock.terminate();lock.wait(timeout=5)
wait(lambda:c.call('GET','/api/v1/users/me',actor=a,schema='User'))
wait(lambda:not firing('APIHighErrorRate'),timeout=70)
data=wait(lambda:trace(error_trace))
assert any(s['name']=='db.transaction' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in spans(data))
result['db_lock'].update(resolved_at=time.time(),trace=data,
diagnosis='Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck.')
save('incidents.json',result)
print('PASS: worker-stop and real DB-lock incidents, local firing/resolved and causal DB traces.')
def collector_metrics():
with OPENER.open(COLLECTOR+'/metrics',timeout=5) as r:raw=r.read().decode()
result={}
for line in raw.splitlines():
if line.startswith('otelcol_') and not line.startswith('#'):
name=line.split('{')[0].split(' ')[0]
try:result[name]=result.get(name,0)+float(line.rsplit(' ',1)[-1])
except ValueError:pass
return result
def backend_outage():
c=Client(BASE);a,b,e,chat=c.bootstrap();before=collector_metrics()
cid=compose('ps','-q','otel-collector')
samples=[];messages=[];attachments=[];start=time.time()
compose('stop','tempo')
try:
for i in range(45):
for _ in range(3):
messages.append(c.message(a,chat,'synthetic-backend-outage')['json']['id'])
if i%10==0:
att,_=c.attachment(a,chat);attachments.append(c.wait_ready(a,att)['id'])
stat=json.loads(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',cid],text=True))
samples.append({'elapsed':time.time()-start,'memory':stat['MemUsage'],'cpu':stat['CPUPerc']})
time.sleep(1)
during=collector_metrics()
assert during.get('otelcol_exporter_send_failed_spans_total',0)>before.get('otelcol_exporter_send_failed_spans_total',0)
assert during.get('otelcol_exporter_queue_size',0)<=128*2
history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100',actor=b,schema='MessagePage')['json']
got={m['id'] for m in history['items']}
if history['next_cursor']:
history2=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+history['next_cursor'],actor=b,schema='MessagePage')['json']
got.update(m['id'] for m in history2['items'])
assert set(messages)<=got
finally:compose('up','-d','--no-deps','tempo')
wait(lambda:get(TEMPO+'/api/search'))
tid=uuid.uuid4().hex
upload(c,a,chat,tid)
wait(lambda:trace(tid))
save('backend-outage.json',{'duration_seconds':time.time()-start,'confirmed_messages':len(messages),
'confirmed_attachments_ready':len(attachments),'all_confirmed_messages_retained':True,
'collector_samples':samples,'before':before,'during':during,'recovered_trace_id':tid})
print('PASS: Tempo unavailable, finite Collector buffers/drops, API writes/jobs retained, fresh trace after recovery.')
if __name__=='__main__':
parser=argparse.ArgumentParser()
parser.add_argument('phase',choices=['proof','incidents','backend'])
args=parser.parse_args()
{'proof':proof,'incidents':incidents,'backend':backend_outage}[args.phase]()