Files
mtusi2026lab6/scripts/prove-first-spike.py
T
2026-11-17 13:08:13 +03:00

32 lines
2.4 KiB
Python

#!/usr/bin/env python3
"""Confirm that Prometheus sees the first 5xx spike after zero-series initialization."""
import concurrent.futures,json,subprocess,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client,check
from observations import query,scalar
C=[str(ROOT/'scripts/compose.sh')]
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
before=query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}')
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',"SET application_name='lab6_counter_probe'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(5); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
def request():return c.call('GET','/api/v1/chats/'+chat['id']+'/messages',expected=[200,500,503],actor=a)['status']
try:
end=time.time()+5
while time.time()<end:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_counter_probe' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
time.sleep(.05)
futures=[]
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
start=time.monotonic()
for i in range(50):
time.sleep(max(0,start+i*.1-time.monotonic()));futures.append(pool.submit(request))
statuses=[f.result() for f in futures]
finally:
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_counter_probe'");proc.wait(timeout=10)
time.sleep(15)
increase=scalar('sum(increase(messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}[1m]))')
check(increase is not None and increase>0,'first spike missing from rate/increase')
out={'at':time.time(),'before':before,'after':query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'),'increase_1m':increase,'statuses':{str(s):statuses.count(s) for s in set(statuses)},'passed':True}
(ROOT/'evidence/first-spike.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: first 5xx spike appears in Prometheus increase:',round(increase,2))