Files
mtusi2026lab7/scripts/lab6_ttr_worker_down.py
T

112 lines
4.4 KiB
Python

#!/usr/bin/env python3
"""Isolated 15-minute worker outage with TTR/queue/alert samples."""
import argparse
import json
import os
import subprocess
import time
import urllib.parse
import urllib.request
from pathlib import Path
from lab7_environment import Environment
from lab7_http import BASE_URL, open_url
ROOT = Path(__file__).resolve().parents[1]
for key in ('HTTP_PROXY','HTTPS_PROXY','ALL_PROXY','http_proxy','https_proxy','all_proxy'):
os.environ.pop(key, None)
def prom(expr):
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
with open_url(url, timeout=10, operator=True) as response:
result = json.loads(response.read()).get('data', {}).get('result', [])
return result[0]['value'][1] if result else 'MISSING'
def alerts():
with open_url('https://localhost:9090/api/v1/alerts', timeout=10, operator=True) as response:
data = json.loads(response.read()).get('data', {}).get('alerts', [])
return [(a['labels'].get('alertname', ''), a['state']) for a in data]
def compose(*args, check=True):
return Environment().compose(*args, check=check)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--duration', type=int, default=930, help='sampling window seconds')
parser.add_argument('--loads', type=int, default=1, help='initial load users')
parser.add_argument('--spawn-every', type=int, default=90, help='start a new load user every N seconds')
args = parser.parse_args()
print('UTC', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), flush=True)
print('HEAD', subprocess.run(['git','rev-parse','HEAD'], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip(), flush=True)
print('$ /api/v1/rules (contains or vector(0))', flush=True)
with open_url('https://localhost:9090/api/v1/rules', timeout=10, operator=True) as response:
rules = response.read().decode()
print('or_vector_0=', 'or vector(0)' in rules, flush=True)
print('json_len=', len(rules), flush=True)
print('$ pre ALERTS', alerts(), flush=True)
loads = []
def start_load():
index = len(loads)
loads.append(subprocess.Popen([
'python3', str(ROOT/'scripts/lab6_load.py'), '--base-url', BASE_URL,
'--rate', '0.2', '--duration', '1000', '--upload-ratio', '1',
'--deadline', '60', '--output', '/tmp/lab6-ttr-worker-load-%d.json' % index,
], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL))
for _ in range(args.loads):
start_load()
time.sleep(5)
print('$ stop worker', flush=True)
print(compose('stop','worker').stdout.strip(), flush=True)
started = time.monotonic()
while time.monotonic() - started < args.duration:
time.sleep(30)
elapsed = int(time.monotonic() - started)
if args.spawn_every > 0 and elapsed and elapsed % args.spawn_every == 0:
start_load()
print(
't=%ss '%elapsed +
'ratio5=' + prom('slo:job_ttr_bad_ratio:5m') +
' ratio1h=' + prom('slo:job_ttr_bad_ratio:1h') +
' missed_total=' + prom('sum(messenger_jobs_ttr_missed_total)') +
' missed_inc5=' + prom('sum(increase(messenger_jobs_ttr_missed_total[5m]))') +
' bucket_count=' + prom('count(messenger_job_time_to_ready_seconds_bucket)') +
' overdue=' + prom('max(messenger_jobs_overdue)') +
' oldest=' + prom('max(messenger_queue_oldest_job_age_seconds)') +
' alerts=' + json.dumps(alerts()),
flush=True,
)
print('$ start worker', flush=True)
print(compose('start','worker').stdout.strip(), flush=True)
for elapsed in (30, 60, 120, 180, 240, 300):
time.sleep(30 if elapsed == 30 else 30)
print(
'recovery t=%ss '%elapsed +
'ratio5=' + prom('slo:job_ttr_bad_ratio:5m') +
' ratio1h=' + prom('slo:job_ttr_bad_ratio:1h') +
' missed_total=' + prom('sum(messenger_jobs_ttr_missed_total)') +
' overdue=' + prom('max(messenger_jobs_overdue)') +
' oldest=' + prom('max(messenger_queue_oldest_job_age_seconds)') +
' alerts=' + json.dumps(alerts()),
flush=True,
)
for load in loads:
load.terminate()
for load in loads:
try:
load.wait(timeout=15)
except subprocess.TimeoutExpired:
load.kill()
if __name__ == '__main__':
main()