forked from mtusi2026/mtusi2026lab6
208 lines
12 KiB
Python
208 lines
12 KiB
Python
"""Track 2: real-time SLO alert experiment against the current synthetic lab6 stack."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(ROOT / 'tests'), str(ROOT)]
|
|
from client import Client
|
|
from research.observations import compose, logs, proxy, query, save, spans, sql, trace, wait
|
|
|
|
|
|
def worker_evidence(requests, phases, started):
|
|
stopped = next(p for p in phases if p['name'] == 'worker_stopped')
|
|
restored = next(p for p in phases if p['name'] == 'after_worker_recovery')
|
|
upload = next(r for r in requests if r['kind'] == 'upload' and r['status'] == 202
|
|
and stopped['active_at'] + 30 < r['utc'] < stopped['end'] - 30)
|
|
stored = trace(upload['trace_id'])
|
|
chain = spans(stored)
|
|
consumers = [s for s in chain if s['name'] == 'attachments process']
|
|
assert consumers, 'Accepted upload must eventually have a real consumer span'
|
|
consumed_at = min(int(s['startTimeUnixNano']) / 1e9 for s in consumers)
|
|
assert consumed_at >= restored['active_at'] - 2, 'Job was consumed while worker was stopped'
|
|
assert {'s3.get_object', 's3.put_object', 'postgresql.UPDATE'} <= {s['name'] for s in chain}
|
|
save('lab6-worker-incident-trace.json', stored)
|
|
save('lab6-worker-incident-logs.json', logs(
|
|
'{service_name=~"messenger-api|messenger-worker"} |= "' + upload['trace_id'] + '"', start=started))
|
|
return {'trace_id': upload['trace_id'], 'request_id': upload['request_id'],
|
|
'accept_to_consume_seconds': round(consumed_at - upload['utc'], 2),
|
|
'consumer_after_restore': True}
|
|
|
|
|
|
def main():
|
|
client = Client()
|
|
alice, _, _, chat = client.bootstrap()
|
|
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
started = time.time()
|
|
stop = threading.Event()
|
|
lock = threading.Lock()
|
|
requests, samples, phases = [], [], []
|
|
skipped = 0
|
|
phase_name = 'baseline'
|
|
pending = set()
|
|
|
|
def send(kind):
|
|
now = time.time()
|
|
name_at_start = phase_name
|
|
try:
|
|
if kind == 'message':
|
|
r = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201,500,503],
|
|
actor=alice,body={'text':'synthetic SLO experiment'},headers={'Idempotency-Key':uuid4().hex},schema='Message')
|
|
else:
|
|
r = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', [202,500,503],
|
|
actor=alice,raw=png,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
row = {'utc':now,'phase':name_at_start,'kind':kind,'status':r['status'],
|
|
'duration_seconds':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id',''),
|
|
'request_id':r['headers']['X-Request-Id']}
|
|
except Exception as error:
|
|
row = {'utc':now,'phase':name_at_start,'kind':kind,'status':0,'error_type':type(error).__name__,
|
|
'duration_seconds':time.time()-now}
|
|
with lock:
|
|
requests.append(row)
|
|
|
|
def load():
|
|
nonlocal skipped
|
|
tick = 0
|
|
next_tick = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
|
while not stop.is_set():
|
|
pending.difference_update({f for f in pending if f.done()})
|
|
kinds = ['message'] + (['upload'] if tick % 4 == 0 else [])
|
|
for kind in kinds:
|
|
if len(pending) >= 16:
|
|
skipped += 1
|
|
else:
|
|
pending.add(pool.submit(send, kind))
|
|
tick += 1
|
|
next_tick += .5
|
|
stop.wait(max(0,next_tick-time.monotonic()))
|
|
|
|
def phase(name, seconds, action=None):
|
|
nonlocal phase_name
|
|
phase_name = name
|
|
row = {'name':name,'start':time.time(),'planned_seconds':seconds}
|
|
print('PHASE:', name, seconds, 'seconds', flush=True)
|
|
if action:
|
|
action()
|
|
row['active_at'] = time.time()
|
|
deadline = row['start'] + seconds
|
|
while time.time() < deadline:
|
|
alerts = proxy('prometheus','/api/v1/alerts')['data']['alerts']
|
|
samples.append({'utc':time.time(),'phase':name,'alerts':[
|
|
{'name':a['labels']['alertname'],'state':a['state'],'activeAt':a['activeAt']} for a in alerts],
|
|
'queue_depth':query('max(messenger_queue_depth)'),
|
|
'queue_age':query('max(messenger_queue_oldest_age_seconds)'),
|
|
'api_bad_30s':query('lab6:api_bad_ratio:30s'),
|
|
'api_bad_2m':query('lab6:api_bad_ratio:2m')})
|
|
time.sleep(min(5,max(0,deadline-time.time())))
|
|
row['end'] = time.time()
|
|
phases.append(row)
|
|
|
|
def db_lock(seconds):
|
|
# Named session and bounded pg_sleep: the fault self-releases even if the driver exits.
|
|
p = subprocess.Popen(['docker','compose','exec','-T','-e','PGAPPNAME=lab6-incident','postgres',
|
|
'psql','-v','ON_ERROR_STOP=1','-U','messenger','-d','messenger','-c',
|
|
f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;"],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
wait(lambda: sql("SELECT count(*) FROM pg_stat_activity WHERE application_name='lab6-incident' AND wait_event='PgSleep'") == '1',timeout=15)
|
|
return p
|
|
|
|
thread = threading.Thread(target=load)
|
|
result = {'utc':datetime.now(timezone.utc).isoformat(),'hypothesis':
|
|
'Multiwindow burn alerts ignore a 4s spike, detect sustained DB/worker degradation and resolve after recovery.',
|
|
'profile':{'message_arrival_per_second':2,'image_arrival_per_second':.5,'max_inflight':16,
|
|
'sampling':1.0,'retries':0,'slo_window_seconds':3600,'api_slo':.99,'ready_slo':.95,'ready_deadline_seconds':30},
|
|
'docker':subprocess.check_output(['docker','info','--format','{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}'],text=True).strip()}
|
|
thread.start()
|
|
try:
|
|
phase('baseline',120)
|
|
phase('short_db_spike',4,lambda:db_lock(4))
|
|
phase('after_short_spike',60)
|
|
phase('sustained_db_lock',90,lambda:db_lock(90))
|
|
# Read only saved signals to retain the diagnosis: error trace + correlated log.
|
|
failing = next(r for r in reversed(requests) if r['status'] >= 500 and r.get('trace_id'))
|
|
time.sleep(3)
|
|
saved_trace = trace(failing['trace_id'])
|
|
save('lab6-db-incident-trace.json',saved_trace)
|
|
save('lab6-db-incident-logs.json',logs('{service_name="messenger-api"} |= "'+failing['trace_id']+'"',start=started))
|
|
result['db_diagnosis'] = {'trace_id':failing['trace_id'],'request_id':failing['request_id'],
|
|
'slow_error_spans':[{'name':s['name'],'status':s.get('status'),
|
|
'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,
|
|
'attributes':s.get('attributes',[])} for s in spans(saved_trace) if s.get('status',{}).get('code') in [2,'STATUS_CODE_ERROR']]}
|
|
phase('after_db_recovery',100)
|
|
phase('worker_stopped',120,lambda:compose('stop','--timeout','5','worker'))
|
|
phase('after_worker_recovery',120,lambda:compose('start','worker'))
|
|
result['worker_diagnosis'] = worker_evidence(requests, phases, started)
|
|
assert all(r['status'] in {201,202,500,503} for r in requests), 'Transport/client errors occurred'
|
|
baseline_firing = {a['name'] for s in samples if s['phase']=='baseline' for a in s['alerts'] if a['state']=='firing'}
|
|
assert not baseline_firing, baseline_firing
|
|
spike_firing = {a['name'] for s in samples if s['phase'] in {'short_db_spike','after_short_spike'} for a in s['alerts'] if a['state']=='firing'}
|
|
assert not any('Burn' in a for a in spike_firing), spike_firing
|
|
spike_peak = max([float(v['value'][1]) for s in samples
|
|
if s['phase'] in {'short_db_spike','after_short_spike'} for v in s['api_bad_30s']] or [0])
|
|
assert spike_peak > 0, 'The real short spike must be visible in the stored error ratio'
|
|
result['short_spike_observed_error_ratio_peak'] = spike_peak
|
|
firing = {a['name'] for s in samples for a in s['alerts'] if a['state']=='firing'}
|
|
expected = {'APIAvailabilityFastBurn','APIAvailabilitySlowBurn','AttachmentReadyFastBurn',
|
|
'AttachmentReadySlowBurn','AttachmentQueueLag','APIHighErrorRate'}
|
|
assert expected <= firing, expected-firing
|
|
assert not [a for a in samples[-1]['alerts'] if a['state']=='firing'], samples[-1]
|
|
result['passed']=True
|
|
result['false_positives']={'baseline':len(baseline_firing),'burn_alerts_after_short_spike':0,
|
|
'single_window_alerts_after_short_spike':sorted(spike_firing)}
|
|
finally:
|
|
stop.set(); thread.join(timeout=20)
|
|
# End the bounded lock early on interruption; restore the worker in all cases.
|
|
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6-incident'")
|
|
compose('start','worker')
|
|
ended=time.time()
|
|
result.update({'start':started,'end':ended,'duration_seconds':ended-started,'phases':phases,
|
|
'samples':samples,'requests':requests,'skipped_arrivals':skipped})
|
|
summary={}
|
|
for p in phases:
|
|
rows=[r for r in requests if p['start']<=r['utc']<p['end']]
|
|
latencies=sorted(r['duration_seconds']*1000 for r in rows)
|
|
summary[p['name']]={'requests':len(rows),'5xx':sum(r['status']>=500 for r in rows),
|
|
'successful_rps':sum(200<=r['status']<300 for r in rows)/(p['end']-p['start']),
|
|
**{f'p{q}_ms':round(latencies[min(len(latencies)-1,int(len(latencies)*q/100))],3)
|
|
for q in [50,95,99] if latencies}}
|
|
result['phase_summary']=summary
|
|
# Actual detection and resolution times, measured from observed Prometheus states.
|
|
timings={}
|
|
for prefix,incident,recovery in [('API','sustained_db_lock','after_db_recovery'),('Attachment','worker_stopped','after_worker_recovery')]:
|
|
ip=next((p for p in phases if p['name']==incident),None)
|
|
rp=next((p for p in phases if p['name']==recovery),None)
|
|
if not ip or not rp: continue
|
|
names={a['name'] for s in samples for a in s['alerts'] if a['name'].startswith(prefix) and a['state']=='firing'}
|
|
for name in names:
|
|
fired=[s['utc'] for s in samples if s['utc']>=ip['start'] and any(a['name']==name and a['state']=='firing' for a in s['alerts'])]
|
|
if not fired: continue
|
|
resolved=next((s['utc'] for s in samples if s['utc']>=max(rp['active_at'],fired[0]) and not any(a['name']==name and a['state']=='firing' for a in s['alerts'])),None)
|
|
timings[name]={'detection_seconds':round(fired[0]-ip['active_at'],2),
|
|
'resolution_after_restore_seconds':round(resolved-rp['active_at'],2) if resolved else None}
|
|
result['alert_timings']=timings
|
|
expressions={'http_rps':'sum(rate(messenger_http_requests_total[1m]))',
|
|
'http_p95':'histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))',
|
|
'queue_depth':'max(messenger_queue_depth)','queue_age':'max(messenger_queue_oldest_age_seconds)',
|
|
'db_pool':'sum by(state)(messenger_db_pool)','worker_busy':'sum(messenger_worker_busy)',
|
|
'worker_capacity':'sum(messenger_worker_capacity)',
|
|
'cpu':'sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))',
|
|
'rss':'sum by(service_name)(messenger_process_memory_bytes)',
|
|
'api_bad':'lab6:api_bad_ratio:2m','ready_bad':'lab6:ready_bad_ratio:2m','alerts':'ALERTS{alertstate="firing"}'}
|
|
series={name:proxy('prometheus','/api/v1/query_range',query=expr,start=started,end=ended,step=5) for name,expr in expressions.items()}
|
|
save('lab6-slo-series.json',series)
|
|
save('lab6-slo-research.json',result)
|
|
print(json.dumps({k:v for k,v in result.items() if k not in {'requests','samples'}},ensure_ascii=False,indent=2),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|