Files
mtusi2026lab6/research/slo.py

208 lines
12 KiB
Python

"""Track 2: real-time SLO alert experiment against the current synthetic lab6 stack."""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'tests'), str(ROOT)]
from client import Client
from research.observations import compose, logs, proxy, query, save, spans, sql, trace, wait
def worker_evidence(requests, phases, started):
stopped = next(p for p in phases if p['name'] == 'worker_stopped')
restored = next(p for p in phases if p['name'] == 'after_worker_recovery')
upload = next(r for r in requests if r['kind'] == 'upload' and r['status'] == 202
and stopped['active_at'] + 30 < r['utc'] < stopped['end'] - 30)
stored = trace(upload['trace_id'])
chain = spans(stored)
consumers = [s for s in chain if s['name'] == 'attachments process']
assert consumers, 'Accepted upload must eventually have a real consumer span'
consumed_at = min(int(s['startTimeUnixNano']) / 1e9 for s in consumers)
assert consumed_at >= restored['active_at'] - 2, 'Job was consumed while worker was stopped'
assert {'s3.get_object', 's3.put_object', 'postgresql.UPDATE'} <= {s['name'] for s in chain}
save('lab6-worker-incident-trace.json', stored)
save('lab6-worker-incident-logs.json', logs(
'{service_name=~"messenger-api|messenger-worker"} |= "' + upload['trace_id'] + '"', start=started))
return {'trace_id': upload['trace_id'], 'request_id': upload['request_id'],
'accept_to_consume_seconds': round(consumed_at - upload['utc'], 2),
'consumer_after_restore': True}
def main():
client = Client()
alice, _, _, chat = client.bootstrap()
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
started = time.time()
stop = threading.Event()
lock = threading.Lock()
requests, samples, phases = [], [], []
skipped = 0
phase_name = 'baseline'
pending = set()
def send(kind):
now = time.time()
name_at_start = phase_name
try:
if kind == 'message':
r = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201,500,503],
actor=alice,body={'text':'synthetic SLO experiment'},headers={'Idempotency-Key':uuid4().hex},schema='Message')
else:
r = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', [202,500,503],
actor=alice,raw=png,headers={'Content-Type':'image/png'},schema='Attachment')
row = {'utc':now,'phase':name_at_start,'kind':kind,'status':r['status'],
'duration_seconds':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id',''),
'request_id':r['headers']['X-Request-Id']}
except Exception as error:
row = {'utc':now,'phase':name_at_start,'kind':kind,'status':0,'error_type':type(error).__name__,
'duration_seconds':time.time()-now}
with lock:
requests.append(row)
def load():
nonlocal skipped
tick = 0
next_tick = time.monotonic()
with ThreadPoolExecutor(max_workers=16) as pool:
while not stop.is_set():
pending.difference_update({f for f in pending if f.done()})
kinds = ['message'] + (['upload'] if tick % 4 == 0 else [])
for kind in kinds:
if len(pending) >= 16:
skipped += 1
else:
pending.add(pool.submit(send, kind))
tick += 1
next_tick += .5
stop.wait(max(0,next_tick-time.monotonic()))
def phase(name, seconds, action=None):
nonlocal phase_name
phase_name = name
row = {'name':name,'start':time.time(),'planned_seconds':seconds}
print('PHASE:', name, seconds, 'seconds', flush=True)
if action:
action()
row['active_at'] = time.time()
deadline = row['start'] + seconds
while time.time() < deadline:
alerts = proxy('prometheus','/api/v1/alerts')['data']['alerts']
samples.append({'utc':time.time(),'phase':name,'alerts':[
{'name':a['labels']['alertname'],'state':a['state'],'activeAt':a['activeAt']} for a in alerts],
'queue_depth':query('max(messenger_queue_depth)'),
'queue_age':query('max(messenger_queue_oldest_age_seconds)'),
'api_bad_30s':query('lab6:api_bad_ratio:30s'),
'api_bad_2m':query('lab6:api_bad_ratio:2m')})
time.sleep(min(5,max(0,deadline-time.time())))
row['end'] = time.time()
phases.append(row)
def db_lock(seconds):
# Named session and bounded pg_sleep: the fault self-releases even if the driver exits.
p = subprocess.Popen(['docker','compose','exec','-T','-e','PGAPPNAME=lab6-incident','postgres',
'psql','-v','ON_ERROR_STOP=1','-U','messenger','-d','messenger','-c',
f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;"],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
wait(lambda: sql("SELECT count(*) FROM pg_stat_activity WHERE application_name='lab6-incident' AND wait_event='PgSleep'") == '1',timeout=15)
return p
thread = threading.Thread(target=load)
result = {'utc':datetime.now(timezone.utc).isoformat(),'hypothesis':
'Multiwindow burn alerts ignore a 4s spike, detect sustained DB/worker degradation and resolve after recovery.',
'profile':{'message_arrival_per_second':2,'image_arrival_per_second':.5,'max_inflight':16,
'sampling':1.0,'retries':0,'slo_window_seconds':3600,'api_slo':.99,'ready_slo':.95,'ready_deadline_seconds':30},
'docker':subprocess.check_output(['docker','info','--format','{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}'],text=True).strip()}
thread.start()
try:
phase('baseline',120)
phase('short_db_spike',4,lambda:db_lock(4))
phase('after_short_spike',60)
phase('sustained_db_lock',90,lambda:db_lock(90))
# Read only saved signals to retain the diagnosis: error trace + correlated log.
failing = next(r for r in reversed(requests) if r['status'] >= 500 and r.get('trace_id'))
time.sleep(3)
saved_trace = trace(failing['trace_id'])
save('lab6-db-incident-trace.json',saved_trace)
save('lab6-db-incident-logs.json',logs('{service_name="messenger-api"} |= "'+failing['trace_id']+'"',start=started))
result['db_diagnosis'] = {'trace_id':failing['trace_id'],'request_id':failing['request_id'],
'slow_error_spans':[{'name':s['name'],'status':s.get('status'),
'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,
'attributes':s.get('attributes',[])} for s in spans(saved_trace) if s.get('status',{}).get('code') in [2,'STATUS_CODE_ERROR']]}
phase('after_db_recovery',100)
phase('worker_stopped',120,lambda:compose('stop','--timeout','5','worker'))
phase('after_worker_recovery',120,lambda:compose('start','worker'))
result['worker_diagnosis'] = worker_evidence(requests, phases, started)
assert all(r['status'] in {201,202,500,503} for r in requests), 'Transport/client errors occurred'
baseline_firing = {a['name'] for s in samples if s['phase']=='baseline' for a in s['alerts'] if a['state']=='firing'}
assert not baseline_firing, baseline_firing
spike_firing = {a['name'] for s in samples if s['phase'] in {'short_db_spike','after_short_spike'} for a in s['alerts'] if a['state']=='firing'}
assert not any('Burn' in a for a in spike_firing), spike_firing
spike_peak = max([float(v['value'][1]) for s in samples
if s['phase'] in {'short_db_spike','after_short_spike'} for v in s['api_bad_30s']] or [0])
assert spike_peak > 0, 'The real short spike must be visible in the stored error ratio'
result['short_spike_observed_error_ratio_peak'] = spike_peak
firing = {a['name'] for s in samples for a in s['alerts'] if a['state']=='firing'}
expected = {'APIAvailabilityFastBurn','APIAvailabilitySlowBurn','AttachmentReadyFastBurn',
'AttachmentReadySlowBurn','AttachmentQueueLag','APIHighErrorRate'}
assert expected <= firing, expected-firing
assert not [a for a in samples[-1]['alerts'] if a['state']=='firing'], samples[-1]
result['passed']=True
result['false_positives']={'baseline':len(baseline_firing),'burn_alerts_after_short_spike':0,
'single_window_alerts_after_short_spike':sorted(spike_firing)}
finally:
stop.set(); thread.join(timeout=20)
# End the bounded lock early on interruption; restore the worker in all cases.
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6-incident'")
compose('start','worker')
ended=time.time()
result.update({'start':started,'end':ended,'duration_seconds':ended-started,'phases':phases,
'samples':samples,'requests':requests,'skipped_arrivals':skipped})
summary={}
for p in phases:
rows=[r for r in requests if p['start']<=r['utc']<p['end']]
latencies=sorted(r['duration_seconds']*1000 for r in rows)
summary[p['name']]={'requests':len(rows),'5xx':sum(r['status']>=500 for r in rows),
'successful_rps':sum(200<=r['status']<300 for r in rows)/(p['end']-p['start']),
**{f'p{q}_ms':round(latencies[min(len(latencies)-1,int(len(latencies)*q/100))],3)
for q in [50,95,99] if latencies}}
result['phase_summary']=summary
# Actual detection and resolution times, measured from observed Prometheus states.
timings={}
for prefix,incident,recovery in [('API','sustained_db_lock','after_db_recovery'),('Attachment','worker_stopped','after_worker_recovery')]:
ip=next((p for p in phases if p['name']==incident),None)
rp=next((p for p in phases if p['name']==recovery),None)
if not ip or not rp: continue
names={a['name'] for s in samples for a in s['alerts'] if a['name'].startswith(prefix) and a['state']=='firing'}
for name in names:
fired=[s['utc'] for s in samples if s['utc']>=ip['start'] and any(a['name']==name and a['state']=='firing' for a in s['alerts'])]
if not fired: continue
resolved=next((s['utc'] for s in samples if s['utc']>=max(rp['active_at'],fired[0]) and not any(a['name']==name and a['state']=='firing' for a in s['alerts'])),None)
timings[name]={'detection_seconds':round(fired[0]-ip['active_at'],2),
'resolution_after_restore_seconds':round(resolved-rp['active_at'],2) if resolved else None}
result['alert_timings']=timings
expressions={'http_rps':'sum(rate(messenger_http_requests_total[1m]))',
'http_p95':'histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))',
'queue_depth':'max(messenger_queue_depth)','queue_age':'max(messenger_queue_oldest_age_seconds)',
'db_pool':'sum by(state)(messenger_db_pool)','worker_busy':'sum(messenger_worker_busy)',
'worker_capacity':'sum(messenger_worker_capacity)',
'cpu':'sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))',
'rss':'sum by(service_name)(messenger_process_memory_bytes)',
'api_bad':'lab6:api_bad_ratio:2m','ready_bad':'lab6:ready_bad_ratio:2m','alerts':'ALERTS{alertstate="firing"}'}
series={name:proxy('prometheus','/api/v1/query_range',query=expr,start=started,end=ended,step=5) for name,expr in expressions.items()}
save('lab6-slo-series.json',series)
save('lab6-slo-research.json',result)
print(json.dumps({k:v for k,v in result.items() if k not in {'requests','samples'}},ensure_ascii=False,indent=2),flush=True)
if __name__=='__main__':
main()