forked from mtusi2026/mtusi2026lab7
112 lines
4.4 KiB
Python
112 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated 15-minute worker outage with TTR/queue/alert samples."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from lab7_environment import Environment
|
|
from lab7_http import BASE_URL, open_url
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
for key in ('HTTP_PROXY','HTTPS_PROXY','ALL_PROXY','http_proxy','https_proxy','all_proxy'):
|
|
os.environ.pop(key, None)
|
|
|
|
|
|
def prom(expr):
|
|
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
|
|
with open_url(url, timeout=10, operator=True) as response:
|
|
result = json.loads(response.read()).get('data', {}).get('result', [])
|
|
return result[0]['value'][1] if result else 'MISSING'
|
|
|
|
|
|
def alerts():
|
|
with open_url('https://localhost:9090/api/v1/alerts', timeout=10, operator=True) as response:
|
|
data = json.loads(response.read()).get('data', {}).get('alerts', [])
|
|
return [(a['labels'].get('alertname', ''), a['state']) for a in data]
|
|
|
|
|
|
def compose(*args, check=True):
|
|
return Environment().compose(*args, check=check)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--duration', type=int, default=930, help='sampling window seconds')
|
|
parser.add_argument('--loads', type=int, default=1, help='initial load users')
|
|
parser.add_argument('--spawn-every', type=int, default=90, help='start a new load user every N seconds')
|
|
args = parser.parse_args()
|
|
print('UTC', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), flush=True)
|
|
print('HEAD', subprocess.run(['git','rev-parse','HEAD'], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip(), flush=True)
|
|
print('$ /api/v1/rules (contains or vector(0))', flush=True)
|
|
with open_url('https://localhost:9090/api/v1/rules', timeout=10, operator=True) as response:
|
|
rules = response.read().decode()
|
|
print('or_vector_0=', 'or vector(0)' in rules, flush=True)
|
|
print('json_len=', len(rules), flush=True)
|
|
print('$ pre ALERTS', alerts(), flush=True)
|
|
|
|
loads = []
|
|
|
|
def start_load():
|
|
index = len(loads)
|
|
loads.append(subprocess.Popen([
|
|
'python3', str(ROOT/'scripts/lab6_load.py'), '--base-url', BASE_URL,
|
|
'--rate', '0.2', '--duration', '1000', '--upload-ratio', '1',
|
|
'--deadline', '60', '--output', '/tmp/lab6-ttr-worker-load-%d.json' % index,
|
|
], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL))
|
|
|
|
for _ in range(args.loads):
|
|
start_load()
|
|
time.sleep(5)
|
|
print('$ stop worker', flush=True)
|
|
print(compose('stop','worker').stdout.strip(), flush=True)
|
|
|
|
started = time.monotonic()
|
|
while time.monotonic() - started < args.duration:
|
|
time.sleep(30)
|
|
elapsed = int(time.monotonic() - started)
|
|
if args.spawn_every > 0 and elapsed and elapsed % args.spawn_every == 0:
|
|
start_load()
|
|
print(
|
|
't=%ss '%elapsed +
|
|
'ratio5=' + prom('slo:job_ttr_bad_ratio:5m') +
|
|
' ratio1h=' + prom('slo:job_ttr_bad_ratio:1h') +
|
|
' missed_total=' + prom('sum(messenger_jobs_ttr_missed_total)') +
|
|
' missed_inc5=' + prom('sum(increase(messenger_jobs_ttr_missed_total[5m]))') +
|
|
' bucket_count=' + prom('count(messenger_job_time_to_ready_seconds_bucket)') +
|
|
' overdue=' + prom('max(messenger_jobs_overdue)') +
|
|
' oldest=' + prom('max(messenger_queue_oldest_job_age_seconds)') +
|
|
' alerts=' + json.dumps(alerts()),
|
|
flush=True,
|
|
)
|
|
|
|
print('$ start worker', flush=True)
|
|
print(compose('start','worker').stdout.strip(), flush=True)
|
|
for elapsed in (30, 60, 120, 180, 240, 300):
|
|
time.sleep(30 if elapsed == 30 else 30)
|
|
print(
|
|
'recovery t=%ss '%elapsed +
|
|
'ratio5=' + prom('slo:job_ttr_bad_ratio:5m') +
|
|
' ratio1h=' + prom('slo:job_ttr_bad_ratio:1h') +
|
|
' missed_total=' + prom('sum(messenger_jobs_ttr_missed_total)') +
|
|
' overdue=' + prom('max(messenger_jobs_overdue)') +
|
|
' oldest=' + prom('max(messenger_queue_oldest_job_age_seconds)') +
|
|
' alerts=' + json.dumps(alerts()),
|
|
flush=True,
|
|
)
|
|
for load in loads:
|
|
load.terminate()
|
|
for load in loads:
|
|
try:
|
|
load.wait(timeout=15)
|
|
except subprocess.TimeoutExpired:
|
|
load.kill()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|