258 lines
13 KiB
Python
258 lines
13 KiB
Python
"""Real OTLP evidence, controlled incidents and bounded telemetry-backend outage."""
|
|
import argparse
|
|
import base64
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from urllib.parse import urlencode
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client, Failure
|
|
|
|
BASE=os.environ.get('BASE_URL','http://127.0.0.1:18093')
|
|
PROM=os.environ.get('PROMETHEUS_URL','http://127.0.0.1:19096')
|
|
LOKI=os.environ.get('LOKI_URL','http://127.0.0.1:13106')
|
|
TEMPO=os.environ.get('TEMPO_URL','http://127.0.0.1:13206')
|
|
COLLECTOR=os.environ.get('COLLECTOR_URL','http://127.0.0.1:18896')
|
|
OPENER=urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
def get(url):
|
|
req=urllib.request.Request(url,headers={'Accept':'application/json'})
|
|
with OPENER.open(req,timeout=10) as response:
|
|
return json.load(response)
|
|
|
|
def prom(query):
|
|
return get(PROM+'/api/v1/query?'+urlencode({'query':query}))['data']['result']
|
|
|
|
def scalar(query):
|
|
rows=prom(query)
|
|
return float(rows[0]['value'][1]) if rows else 0
|
|
|
|
def logs(query, start=None, limit=2000):
|
|
params={'query':query,'start':str(int((start or time.time()-600)*1e9)),
|
|
'end':str(time.time_ns()),'limit':str(limit),'direction':'forward'}
|
|
result=get(LOKI+'/loki/api/v1/query_range?'+urlencode(params))['data']['result']
|
|
return [json.loads(value) for stream in result for stamp,value in stream['values']]
|
|
|
|
def trace(identifier):
|
|
return get(TEMPO+'/api/traces/'+identifier)
|
|
|
|
def spans(data):
|
|
result=[]
|
|
for batch in data.get('batches',data.get('resourceSpans',[])):
|
|
service=next(a['value']['stringValue'] for a in batch['resource']['attributes'] if a['key']=='service.name')
|
|
for scope in batch.get('scopeSpans',[]):
|
|
for span in scope['spans']:
|
|
result.append(dict(span,service=service))
|
|
return result
|
|
|
|
def wait(function,timeout=60):
|
|
end=time.monotonic()+timeout
|
|
while time.monotonic()<end:
|
|
try:
|
|
result=function()
|
|
if result:return result
|
|
except (OSError,ValueError,Failure):
|
|
pass
|
|
time.sleep(.5)
|
|
raise AssertionError('Timed out: '+function.__name__)
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(['docker','compose',*args],text=True).strip()
|
|
|
|
def save(name,data):
|
|
(ROOT/'evidence'/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
|
|
|
|
def upload(c,actor,chat,tid):
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
|
|
actor=actor,raw=raw,headers={'Content-Type':'image/png',
|
|
'traceparent':'00-'+tid+'-0123456789abcdef-01'},schema='Attachment')
|
|
|
|
def proof():
|
|
wait(lambda:get(LOKI+'/loki/api/v1/labels'))
|
|
wait(lambda:get(TEMPO+'/api/search'))
|
|
proof_start=time.time()
|
|
c=Client(BASE);a,b,e,chat=c.bootstrap()
|
|
tid=uuid.uuid4().hex
|
|
marker='SECRET_MESSAGE_CANARY_'+uuid.uuid4().hex
|
|
c.message(a,chat,marker)
|
|
response=upload(c,a,chat,tid)
|
|
ready=c.wait_ready(b,response['json'])
|
|
def completed_trace():
|
|
data=trace(tid)
|
|
return data if any(s['name']=='attachment process' for s in spans(data)) else None
|
|
data=wait(completed_trace)
|
|
rows=spans(data)
|
|
upload_span=next(s for s in rows if s['kind']=='SPAN_KIND_SERVER')
|
|
publish=next(s for s in rows if s['kind']=='SPAN_KIND_PRODUCER')
|
|
consumer=next(s for s in rows if s['kind']=='SPAN_KIND_CONSUMER')
|
|
assert consumer['parentSpanId']==publish['spanId']
|
|
index={s['spanId']:s for s in rows}
|
|
parent=publish
|
|
while parent['spanId']!=upload_span['spanId']:
|
|
parent=index[parent['parentSpanId']]
|
|
assert any(s['service']=='messenger-worker' and s['name'].startswith('s3.') for s in rows)
|
|
assert any(s['service']=='messenger-worker' and s['name']=='db.transaction' for s in rows)
|
|
query='{service_name=~"messenger-.*"} | json | trace_id="'+tid+'"'
|
|
records=wait(lambda: logs(query) if len(logs(query))>=4 else None)
|
|
assert {r['event'] for r in records}>={'http_request','attachment_accepted','job_published','job_processed'}
|
|
assert any(r.get('request_id')==response['headers'].get('X-Request-Id') for r in records)
|
|
assert all(len(r['trace_id'])==32 and len(r['span_id'])==16 for r in records)
|
|
all_logs=logs('{service_name=~"messenger-.*"}',start=proof_start,limit=5000)
|
|
payload=json.dumps(all_logs)+json.dumps(data)
|
|
for value in [marker,a['token'],a['registration']['password'],a['registration']['username'],'private-canary.png']:
|
|
assert value not in payload, 'business/credential canary leaked'
|
|
metric_rows=prom('{service_name=~"messenger-.*"}')
|
|
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','job_id','trace_id','span_id'}
|
|
assert all(not forbidden.intersection(row['metric']) for row in metric_rows)
|
|
names={r['metric']['__name__'] for r in metric_rows}
|
|
assert {'http_server_duration_seconds_bucket','jobs_processed_total','queue_depth','queue_oldest_age_seconds','db_pool_used','worker_capacity'}<=names
|
|
save('trace-proof.json',{'trace_id':tid,'request_id':response['headers'].get('X-Request-Id'),
|
|
'status':ready['status'],'consumer_parent_matches_producer':True,'producer_descends_from_upload':True,
|
|
'api_worker_logs_correlated':True,'secret_canaries_absent':True,'metric_labels_allowlisted':True,
|
|
'trace':data,'logs':records})
|
|
# Retry is a real S3 outage after durable acceptance, not fabricated spans.
|
|
compose('stop','worker')
|
|
retry_tid=uuid.uuid4().hex
|
|
queued=upload(c,a,chat,retry_tid)['json']
|
|
paused=False
|
|
try:
|
|
compose('pause','s3');paused=True
|
|
compose('up','-d','--no-deps','worker')
|
|
def attempted():
|
|
raw=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
|
|
"SELECT attempt FROM attachment_jobs WHERE 1=1 AND attachment_id = '"+queued['id']+"';")
|
|
return int(raw)>=1
|
|
wait(attempted)
|
|
time.sleep(5)
|
|
finally:
|
|
if paused:compose('unpause','s3')
|
|
compose('up','-d','--no-deps','worker')
|
|
wait(lambda:c.call('GET','/health/ready',schema='Health'))
|
|
c.wait_ready(a,queued)
|
|
def retried():
|
|
data=trace(retry_tid)
|
|
return data if len([s for s in spans(data) if s['kind']=='SPAN_KIND_CONSUMER'])>=2 else None
|
|
retry_data=wait(retried)
|
|
retry_records=wait(lambda:logs('{service_name="messenger-worker"} | json | trace_id="'+retry_tid+'" | event="job_retry"'))
|
|
save('retry-trace.json',{'trace_id':retry_tid,'retry_observed':True,'trace':retry_data,'logs':retry_records})
|
|
print('PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.')
|
|
|
|
def alerts():
|
|
return get(PROM+'/api/v1/alerts')['data']['alerts']
|
|
|
|
def firing(name):
|
|
return [a for a in alerts() if a['labels']['alertname']==name and a['state']=='firing']
|
|
|
|
def incidents():
|
|
c=Client(BASE);a,b,e,chat=c.bootstrap();result={}
|
|
start=time.time()
|
|
compose('stop','worker')
|
|
try:
|
|
queued=[c.attachment(a,chat)[0] for _ in range(3)]
|
|
observed=wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')>=3)
|
|
fired=wait(lambda:firing('QueueLag'))
|
|
detected=time.time()
|
|
metrics={q:prom(q) for q in ['queue_depth','queue_oldest_age_seconds','worker_busy','worker_capacity']}
|
|
result['worker_stop']={'injected_at':start,'detected_at':detected,
|
|
'detection_seconds':detected-start,'firing':fired,'metrics':metrics,
|
|
'diagnosis':'Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.'}
|
|
finally:
|
|
compose('up','-d','--no-deps','worker')
|
|
for att in queued:c.wait_ready(a,att)
|
|
wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')==0)
|
|
wait(lambda:not firing('QueueLag'))
|
|
result['worker_stop']['resolved_at']=time.time()
|
|
# Keep readiness SELECT 1 available while real session SQL is blocked.
|
|
lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1',
|
|
'-c',"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;"],
|
|
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
time.sleep(1)
|
|
db_start=time.time();stop=False;errors=[];error_trace=uuid.uuid4().hex
|
|
def traffic():
|
|
while not stop:
|
|
r=c.call('GET','/api/v1/users/me',[200,503],actor=a,
|
|
headers={'traceparent':'00-'+error_trace+'-0123456789abcdef-01'})
|
|
if r['status']==503:errors.append(r['elapsed'])
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
futures=[pool.submit(traffic) for _ in range(4)]
|
|
try:
|
|
fired=wait(lambda:firing('APIHighErrorRate'),timeout=50)
|
|
detected=time.time()
|
|
result['db_lock']={'injected_at':db_start,'detected_at':detected,'detection_seconds':detected-db_start,
|
|
'firing':fired,'error_latencies_seconds':errors.copy(),'trace_id':error_trace}
|
|
lock.wait(timeout=45)
|
|
finally:
|
|
stop=True
|
|
for future in futures:future.result()
|
|
finally:
|
|
stop=True
|
|
if lock.poll() is None:
|
|
lock.terminate();lock.wait(timeout=5)
|
|
wait(lambda:c.call('GET','/api/v1/users/me',actor=a,schema='User'))
|
|
wait(lambda:not firing('APIHighErrorRate'),timeout=70)
|
|
data=wait(lambda:trace(error_trace))
|
|
assert any(s['name']=='db.transaction' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in spans(data))
|
|
result['db_lock'].update(resolved_at=time.time(),trace=data,
|
|
diagnosis='Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck.')
|
|
save('incidents.json',result)
|
|
print('PASS: worker-stop and real DB-lock incidents, local firing/resolved and causal DB traces.')
|
|
|
|
def collector_metrics():
|
|
with OPENER.open(COLLECTOR+'/metrics',timeout=5) as r:raw=r.read().decode()
|
|
result={}
|
|
for line in raw.splitlines():
|
|
if line.startswith('otelcol_') and not line.startswith('#'):
|
|
name=line.split('{')[0].split(' ')[0]
|
|
try:result[name]=result.get(name,0)+float(line.rsplit(' ',1)[-1])
|
|
except ValueError:pass
|
|
return result
|
|
|
|
def backend_outage():
|
|
c=Client(BASE);a,b,e,chat=c.bootstrap();before=collector_metrics()
|
|
cid=compose('ps','-q','otel-collector')
|
|
samples=[];messages=[];attachments=[];start=time.time()
|
|
compose('stop','tempo')
|
|
try:
|
|
for i in range(45):
|
|
for _ in range(3):
|
|
messages.append(c.message(a,chat,'synthetic-backend-outage')['json']['id'])
|
|
if i%10==0:
|
|
att,_=c.attachment(a,chat);attachments.append(c.wait_ready(a,att)['id'])
|
|
stat=json.loads(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',cid],text=True))
|
|
samples.append({'elapsed':time.time()-start,'memory':stat['MemUsage'],'cpu':stat['CPUPerc']})
|
|
time.sleep(1)
|
|
during=collector_metrics()
|
|
assert during.get('otelcol_exporter_send_failed_spans_total',0)>before.get('otelcol_exporter_send_failed_spans_total',0)
|
|
assert during.get('otelcol_exporter_queue_size',0)<=128*2
|
|
history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100',actor=b,schema='MessagePage')['json']
|
|
got={m['id'] for m in history['items']}
|
|
if history['next_cursor']:
|
|
history2=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+history['next_cursor'],actor=b,schema='MessagePage')['json']
|
|
got.update(m['id'] for m in history2['items'])
|
|
assert set(messages)<=got
|
|
finally:compose('up','-d','--no-deps','tempo')
|
|
wait(lambda:get(TEMPO+'/api/search'))
|
|
tid=uuid.uuid4().hex
|
|
upload(c,a,chat,tid)
|
|
wait(lambda:trace(tid))
|
|
save('backend-outage.json',{'duration_seconds':time.time()-start,'confirmed_messages':len(messages),
|
|
'confirmed_attachments_ready':len(attachments),'all_confirmed_messages_retained':True,
|
|
'collector_samples':samples,'before':before,'during':during,'recovered_trace_id':tid})
|
|
print('PASS: Tempo unavailable, finite Collector buffers/drops, API writes/jobs retained, fresh trace after recovery.')
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('phase',choices=['proof','incidents','backend'])
|
|
args=parser.parse_args()
|
|
{'proof':proof,'incidents':incidents,'backend':backend_outage}[args.phase]()
|