145 lines
8.2 KiB
Python
145 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Query real backends and reproduce bounded incidents; no secret values saved."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,check
|
|
from verify_jobs import compose,inside,wait_for,state
|
|
|
|
|
|
def get(url):
|
|
with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def query(expression):
|
|
return get('http://localhost:9090/api/v1/query?'+urllib.parse.urlencode({'query':expression}))['data']['result']
|
|
|
|
|
|
def alerts():
|
|
return get('http://localhost:9090/api/v1/alerts')['data']['alerts']
|
|
|
|
|
|
def alert_state(name,state):
|
|
rows=[a for a in alerts() if a['labels']['alertname']==name]
|
|
return any(a['state']==state for a in rows) if state=='firing' else not rows
|
|
|
|
|
|
def collect_names(trace):
|
|
names=[]
|
|
def walk(value):
|
|
if isinstance(value,dict):
|
|
if 'spanId' in value and 'name' in value: names.append(value['name'])
|
|
for item in value.values(): walk(item)
|
|
elif isinstance(value,list):
|
|
for item in value: walk(item)
|
|
walk(trace)
|
|
return names
|
|
|
|
|
|
def save(name,data):
|
|
(ROOT/'evidence'/name).write_text(json.dumps(data,indent=2)+'\n')
|
|
|
|
|
|
def main():
|
|
c=Client();alice,bob,_,chat=c.bootstrap()
|
|
report={'result':'INCOMPLETE'}
|
|
lock_process=None
|
|
try:
|
|
# Controlled retry in the same causal chain.
|
|
compose('stop','worker')
|
|
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,
|
|
actor=alice,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),headers={'Content-Type':'image/png'},schema='Attachment')
|
|
att=response['json'];trace_id=response['headers']['X-Trace-Id']
|
|
compose('stop','minio');compose('start','worker')
|
|
wait_for(lambda: (lambda s:s['attempt']>=1 and s['status']=='queued')(state(att['id'])),30)
|
|
time.sleep(2)
|
|
compose('start','minio')
|
|
# API re-enters LB after readiness recovers.
|
|
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0,60)
|
|
ready=c.wait_ready(bob,att)
|
|
def chain():
|
|
try:
|
|
trace=get('http://localhost:3200/api/traces/'+trace_id)
|
|
names=collect_names(trace)
|
|
return trace if names.count('attachments.consume')>=2 and all(n in names for n in ['attachments.publish','attachments.process_objects','s3.put','postgres.query']) else None
|
|
except Exception: return None
|
|
trace=wait_for(chain,60)
|
|
save('upload-trace.json',trace)
|
|
logquery='{service_name=~"messenger-.*"} |= "'+trace_id+'"'
|
|
def correlated_logs():
|
|
try:
|
|
result=get('http://localhost:3100/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':logquery,'limit':100,'start':str(time.time_ns()-600_000_000_000)}))
|
|
streams=result['data']['result']
|
|
services={item['stream'].get('service_name') for item in streams}
|
|
return result if {'messenger-api','messenger-worker'}<=services else None
|
|
except Exception: return None
|
|
|
|
logs=wait_for(correlated_logs,60);save('upload-logs.json',logs)
|
|
report['trace']={'trace_id':trace_id,'span_names':collect_names(trace),'retry_attempts':state(att['id'])['attempt'],'api_worker_logs_correlated':True}
|
|
# Incident 1: worker stopped, queued age is measured by relay.
|
|
wait_for(lambda: alert_state('QueueLagGrowing','resolved'),40)
|
|
compose('stop','worker')
|
|
queued,_=c.attachment(alice,chat)
|
|
wait_for(lambda: alert_state('QueueLagGrowing','firing'),70)
|
|
save('queue-alert-firing.json',alerts())
|
|
report['queue_incident']={'oldest_age':query('max(queue_oldest_age_seconds)'),'depth':query('max(queue_depth)'),'alert':'firing'}
|
|
compose('start','worker');c.wait_ready(bob,queued)
|
|
wait_for(lambda: alert_state('QueueLagGrowing','resolved'),30)
|
|
save('queue-alert-resolved.json',alerts());report['queue_incident']['after_restore']='resolved'
|
|
# Incident 2: sessions table lock affects business auth, readiness remains healthy.
|
|
code='import os\nfrom messenger.db import Database\nd=Database();d.open();c=d.pool.getconn()\nc.execute(\'SET statement_timeout=0\')\nc.execute(\'SET idle_in_transaction_session_timeout=0\')\nc.execute("SET application_name=\'lab6-fault-lock\'")\nc.commit()\nc.execute(\'LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE\')\nprint(\'locked\',flush=True)\ntry:\n c.execute(\'SELECT pg_sleep(90)\')\nfinally:\n os._exit(0)\n'
|
|
lock_process=subprocess.Popen(['docker','compose','exec','-T','api','python','-c',code],cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
|
check(lock_process.stdout.readline().strip()=='locked','DB lock was not acquired')
|
|
failures=[]
|
|
started=time.monotonic()
|
|
while time.monotonic()-started<35:
|
|
r=c.call('GET','/api/v1/users/me',503,actor=alice)
|
|
failures.append({'status':r['status'],'elapsed':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id')})
|
|
wait_for(lambda: alert_state('HighApiErrorRate','firing'),40)
|
|
save('db-alert-firing.json',alerts())
|
|
# Terminate exact lock holder via PG primary, not the API container.
|
|
inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}")
|
|
lock_process.wait(timeout=10);lock_process=None
|
|
for _ in range(300): c.call('GET','/api/v1/users/me',actor=alice)
|
|
wait_for(lambda: alert_state('HighApiErrorRate','resolved'),75)
|
|
save('db-alert-resolved.json',alerts())
|
|
report['db_incident']={'failures':failures,'before':'firing','after':'resolved','pool_waiting':query('sum(db_pool_waiting)')}
|
|
# Backend outage must not affect accepted messages or grow queues beyond configuration.
|
|
before=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip())
|
|
compose('stop','tempo','loki')
|
|
count=0;start=time.monotonic()
|
|
while time.monotonic()-start<25:
|
|
c.message(alice,chat,'telemetry outage '+str(count),key=uuid.uuid4().hex);count+=1
|
|
if count%20==0: time.sleep(.1)
|
|
during=query('{__name__=~"otelcol_(receiver_accepted_spans|exporter_sent_spans|exporter_send_failed_spans|exporter_queue_size|exporter_queue_capacity|exporter_enqueue_failed_spans).*"}')
|
|
after=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip())
|
|
compose('start','tempo','loki')
|
|
report['telemetry_outage']={'successful_messages':count,'seconds':round(time.monotonic()-start,3),'collector_before':before,'collector_after':after,'collector_metrics':during,'queue_limit_batches':256,'collector_limit_mib':256}
|
|
# Every provisioned dashboard expression must return a successful Prometheus response.
|
|
dashboard=json.loads((ROOT/'infra/observability/grafana/dashboards/messenger.json').read_text())
|
|
expressions=[]
|
|
for panel in dashboard['panels']:
|
|
for target in panel['targets']:
|
|
values=query(target['expr']);check(values,'No data for '+panel['title'])
|
|
expressions.append({'panel':panel['title'],'expression':target['expr'],'series':len(values)})
|
|
report['dashboard_queries']=expressions
|
|
report['result']='PASS';save('observability.json',report)
|
|
print('PASS: observability.json, traces/logs and firing/resolved alerts',flush=True)
|
|
finally:
|
|
if lock_process is not None:
|
|
inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}")
|
|
compose('start','worker','minio','tempo','loki')
|
|
|
|
|
|
if __name__=='__main__': main()
|