forked from mtusi2026/mtusi2026lab6
32 lines
2.4 KiB
Python
32 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Confirm that Prometheus sees the first 5xx spike after zero-series initialization."""
|
|
import concurrent.futures,json,subprocess,sys,time
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
|
from client import Client,check
|
|
from observations import query,scalar
|
|
C=[str(ROOT/'scripts/compose.sh')]
|
|
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
|
|
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
|
before=query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}')
|
|
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',"SET application_name='lab6_counter_probe'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(5); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
def request():return c.call('GET','/api/v1/chats/'+chat['id']+'/messages',expected=[200,500,503],actor=a)['status']
|
|
try:
|
|
end=time.time()+5
|
|
while time.time()<end:
|
|
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_counter_probe' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
|
|
time.sleep(.05)
|
|
futures=[]
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
|
|
start=time.monotonic()
|
|
for i in range(50):
|
|
time.sleep(max(0,start+i*.1-time.monotonic()));futures.append(pool.submit(request))
|
|
statuses=[f.result() for f in futures]
|
|
finally:
|
|
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_counter_probe'");proc.wait(timeout=10)
|
|
time.sleep(15)
|
|
increase=scalar('sum(increase(messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}[1m]))')
|
|
check(increase is not None and increase>0,'first spike missing from rate/increase')
|
|
out={'at':time.time(),'before':before,'after':query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'),'increase_1m':increase,'statuses':{str(s):statuses.count(s) for s in set(statuses)},'passed':True}
|
|
(ROOT/'evidence/first-spike.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: first 5xx spike appears in Prometheus increase:',round(increase,2))
|