forked from mtusi2026/mtusi2026lab7
395 lines
16 KiB
Python
Executable File
395 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Full Track 2 timeline with fail-fast, preflight and guaranteed restore.
|
|
|
|
Warmup >= 70m, baseline, 2m burst, 15m sustained outage, resolution, TTR and
|
|
false-positive phases. The script records injection/restore timestamps, alert
|
|
transitions and Prometheus snapshots before and after each fault.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from lab7_environment import Environment
|
|
from lab7_http import BASE_URL, open_url
|
|
|
|
for _key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
|
os.environ.pop(_key, None)
|
|
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
|
|
os.environ['no_proxy'] = 'localhost,127.0.0.1,::1'
|
|
# tests/client.py builds its own opener; make it ignore macOS system proxies too.
|
|
urllib.request.getproxies = lambda: {}
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
ACTIVE_LOAD = [None]
|
|
|
|
|
|
def utc(ts=None):
|
|
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(ts if ts is not None else time.time()))
|
|
|
|
|
|
def http_json(url, timeout=15):
|
|
with open_url(url, timeout=timeout, operator=True) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def prom(expr):
|
|
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
|
|
return http_json(url).get('data', {}).get('result', [])
|
|
|
|
|
|
def alerts():
|
|
return http_json('https://localhost:9090/api/v1/alerts').get('data', {}).get('alerts', [])
|
|
|
|
|
|
def compose(*args, check=True):
|
|
result = Environment().compose(*args, check=False)
|
|
(ROOT / 'evidence/lab6/track2').mkdir(parents=True, exist_ok=True)
|
|
with (ROOT / 'evidence/lab6/track2/full-compose.log').open('a') as log:
|
|
log.write(utc() + ' docker compose ' + ' '.join(args) + ' rc=' + str(result.returncode) + '\n')
|
|
if result.stderr:
|
|
log.write(result.stderr[-2000:] + '\n')
|
|
if check and result.returncode != 0:
|
|
raise RuntimeError('docker compose %s failed rc=%d: %s' % (' '.join(args), result.returncode, result.stderr[-500:]))
|
|
return result
|
|
|
|
|
|
def wait_ready(timeout=120):
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
try:
|
|
with open_url(BASE_URL + '/health/ready', timeout=5) as response:
|
|
if response.status == 200:
|
|
return True
|
|
except Exception:
|
|
pass
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
class AlertTimeline:
|
|
def __init__(self, path):
|
|
self.path = path
|
|
self.stop = threading.Event()
|
|
self.active = {}
|
|
self.changes = []
|
|
self.injection = ''
|
|
self.lock = threading.Lock()
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
self.handle = path.open('w', newline='')
|
|
self.writer = csv.writer(self.handle)
|
|
self.writer.writerow(['observed_utc', 'alert', 'state', 'active_at', 'value', 'injection_utc', 'labels'])
|
|
self.thread = threading.Thread(target=self.run, daemon=True)
|
|
|
|
def start(self):
|
|
self.thread.start()
|
|
|
|
def run(self):
|
|
while not self.stop.is_set():
|
|
try:
|
|
current = {}
|
|
for alert in alerts():
|
|
state = alert.get('state', '')
|
|
if state not in ('pending', 'firing'):
|
|
continue
|
|
labels = alert.get('labels', {})
|
|
name = labels.get('alertname', '')
|
|
key = name + '|' + json.dumps(labels, sort_keys=True, separators=(',', ':'))
|
|
current[key] = (name, state, alert.get('activeAt', ''), alert.get('value', ''), labels)
|
|
for key, (name, state, active_at, value, labels) in current.items():
|
|
previous = self.active.get(key)
|
|
if previous is None or previous[1] != state:
|
|
row = [utc(), name, state, active_at, value, self.injection, json.dumps(labels, sort_keys=True, separators=(',', ':'))]
|
|
with self.lock:
|
|
self.changes.append(row)
|
|
self.writer.writerow(row)
|
|
self.handle.flush()
|
|
for key in sorted(set(self.active) - set(current)):
|
|
name, state, _, _, labels = self.active[key]
|
|
if state != 'firing':
|
|
continue
|
|
row = [utc(), name, 'resolved', '', '', self.injection, json.dumps(labels, sort_keys=True, separators=(',', ':'))]
|
|
with self.lock:
|
|
self.changes.append(row)
|
|
self.writer.writerow(row)
|
|
self.handle.flush()
|
|
self.active = current
|
|
except Exception as exc:
|
|
with self.lock:
|
|
self.writer.writerow([utc(), 'POLL_ERROR', str(exc)[:160], '', '', self.injection])
|
|
self.handle.flush()
|
|
self.stop.wait(5)
|
|
|
|
def finish(self):
|
|
self.stop.set()
|
|
self.thread.join(timeout=20)
|
|
self.handle.close()
|
|
return self.changes
|
|
|
|
|
|
def snapshot(out, label):
|
|
values = {}
|
|
errors = []
|
|
for name, expr in {
|
|
'api_error_5m': 'slo:api_error_ratio:5m',
|
|
'api_error_1h': 'slo:api_error_ratio:1h',
|
|
'api_error_30m': 'slo:api_error_ratio:30m',
|
|
'api_error_6h': 'slo:api_error_ratio:6h',
|
|
'ttr_bad_5m': 'slo:job_ttr_bad_ratio:5m',
|
|
'ttr_bad_1h': 'slo:job_ttr_bad_ratio:1h',
|
|
'queue_oldest': 'max(messenger_queue_oldest_job_age_seconds)',
|
|
'queue_depth': 'max(rabbitmq_queue_messages_ready{queue="attachment.process.v1"})',
|
|
'jobs_overdue': 'max(messenger_jobs_overdue)',
|
|
'worker_busy': 'max(messenger_worker_busy)',
|
|
}.items():
|
|
try:
|
|
result = prom(expr)
|
|
values[name] = result[0]['value'][1] if result else None
|
|
except Exception as exc:
|
|
values[name] = None
|
|
errors.append(name + ': ' + str(exc)[:100])
|
|
(out / 'snapshots').mkdir(parents=True, exist_ok=True)
|
|
(out / 'snapshots' / (label + '-' + utc().replace(':', '-') + '.json')).write_text(json.dumps(values, indent=2) + '\n')
|
|
if errors:
|
|
raise RuntimeError('Prometheus snapshot failed: ' + '; '.join(errors))
|
|
return values
|
|
|
|
|
|
def run_load(out, label, duration, ratio=0.2, rate=0.2):
|
|
log_path = out / (label + '-load.log')
|
|
output_path = out / (label + '-load.json')
|
|
cmd = [sys.executable, str(ROOT / 'scripts/lab6_load.py'),
|
|
'--base-url', BASE_URL, '--rate', str(rate),
|
|
'--duration', str(duration), '--upload-ratio', str(ratio),
|
|
'--deadline', '60', '--output', str(output_path)]
|
|
log = log_path.open('w')
|
|
process = subprocess.Popen(cmd, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT)
|
|
ACTIVE_LOAD[0] = (process, log, output_path)
|
|
return process, log, output_path
|
|
|
|
|
|
def finish_load(process, log, output_path, label, expected_s, allow_failures=False):
|
|
try:
|
|
rc = process.wait(timeout=expected_s + 300)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
rc = process.wait()
|
|
log.close()
|
|
ACTIVE_LOAD[0] = None
|
|
if not output_path.exists():
|
|
raise RuntimeError(label + ': load produced no JSON (rc=' + str(rc) + ')')
|
|
summary = json.loads(output_path.read_text())
|
|
minimum = 0.85 * expected_s if expected_s >= 30 else 0.5 * expected_s
|
|
if summary.get('duration_s', 0) < minimum:
|
|
raise RuntimeError(label + ': load ran only ' + str(summary.get('duration_s')) + 's, expected ~' + str(expected_s))
|
|
if not allow_failures and (rc != 0 or summary.get('failures', 0)):
|
|
raise RuntimeError(label + ': load failed rc=' + str(rc) + ' failures=' + str(summary.get('failures')))
|
|
return summary
|
|
|
|
|
|
def restore_stand():
|
|
compose('unpause', 'postgres', check=False)
|
|
compose('start', 'postgres', check=False)
|
|
compose('up', '-d', '--wait', '--no-deps', 'worker', check=False)
|
|
return wait_ready(120)
|
|
|
|
|
|
def phase(out, phases, name, started, injection='', restore=''):
|
|
value = {
|
|
'phase': name,
|
|
'started_utc': utc(started),
|
|
'finished_utc': utc(),
|
|
'duration_s': round(time.time() - started, 1),
|
|
'injection_utc': injection,
|
|
'restore_utc': restore,
|
|
}
|
|
phases.append(value)
|
|
(out / 'phases.json').write_text(json.dumps(phases, indent=2) + '\n')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Full lab6 Track 2 timeline')
|
|
parser.add_argument('--warmup', type=int, default=4200)
|
|
parser.add_argument('--baseline', type=int, default=1200)
|
|
parser.add_argument('--burst', type=int, default=120)
|
|
parser.add_argument('--settle', type=int, default=600)
|
|
parser.add_argument('--sustained', type=int, default=900)
|
|
parser.add_argument('--resolution', type=int, default=3600)
|
|
parser.add_argument('--ttr', type=int, default=900)
|
|
parser.add_argument('--false-positive', type=int, default=1200)
|
|
parser.add_argument('--out', default='evidence/lab6/track2/full')
|
|
args = parser.parse_args()
|
|
out = ROOT / args.out
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
phases = []
|
|
timeline = AlertTimeline(out / 'alerts.csv')
|
|
timeline.start()
|
|
interrupted = False
|
|
exit_code = 0
|
|
try:
|
|
def on_signal(signum, _frame):
|
|
raise KeyboardInterrupt('signal %d' % signum)
|
|
signal.signal(signal.SIGINT, on_signal)
|
|
signal.signal(signal.SIGTERM, on_signal)
|
|
|
|
if not wait_ready(120):
|
|
raise RuntimeError('preflight: /health/ready is not ready')
|
|
preflight = run_load(out, 'preflight', 3, ratio=0.5, rate=1.0)
|
|
finish_load(*preflight, 'preflight', 3, allow_failures=False)
|
|
print(utc(), 'preflight OK; start full track', flush=True)
|
|
|
|
t = time.time()
|
|
load = run_load(out, 'warmup', args.warmup)
|
|
print(utc(), 'warmup started', flush=True)
|
|
finish_load(*load, 'warmup', args.warmup)
|
|
phase(out, phases, 'warmup', t)
|
|
snapshot(out, 'warmup')
|
|
|
|
t = time.time()
|
|
load = run_load(out, 'baseline', args.baseline)
|
|
print(utc(), 'baseline started', flush=True)
|
|
finish_load(*load, 'baseline', args.baseline)
|
|
phase(out, phases, 'baseline', t)
|
|
(out / 'baseline.json').write_text(json.dumps(snapshot(out, 'baseline'), indent=2) + '\n')
|
|
|
|
t = time.time()
|
|
load = run_load(out, 'burst', args.burst + args.settle)
|
|
time.sleep(10)
|
|
injection = utc()
|
|
timeline.injection = injection
|
|
print(utc(), 'burst: pause postgres', flush=True)
|
|
compose('pause', 'postgres')
|
|
time.sleep(min(30, args.burst))
|
|
burst_active = snapshot(out, 'burst-active')
|
|
if args.burst > 30:
|
|
time.sleep(args.burst - 30)
|
|
compose('unpause', 'postgres')
|
|
restore = utc()
|
|
print(utc(), 'burst: unpause postgres', flush=True)
|
|
finish_load(*load, 'burst', args.burst + args.settle, allow_failures=True)
|
|
phase(out, phases, 'burst', t, injection, restore)
|
|
burst_after = snapshot(out, 'burst-after')
|
|
(out / 'burst.json').write_text(json.dumps({'active': burst_active, 'after': burst_after}, indent=2) + '\n')
|
|
|
|
t = time.time()
|
|
load = run_load(out, 'sustained', args.sustained + 300)
|
|
time.sleep(10)
|
|
injection = utc()
|
|
timeline.injection = injection
|
|
print(utc(), 'sustained: stop postgres', flush=True)
|
|
compose('stop', 'postgres')
|
|
time.sleep(min(30, args.sustained))
|
|
sustained_active = snapshot(out, 'sustained-active')
|
|
if args.sustained > 30:
|
|
time.sleep(args.sustained - 30)
|
|
compose('start', 'postgres')
|
|
if not wait_ready(180):
|
|
raise RuntimeError('sustained: readiness did not recover')
|
|
restore = utc()
|
|
print(utc(), 'sustained: postgres restored', flush=True)
|
|
finish_load(*load, 'sustained', args.sustained + 300, allow_failures=True)
|
|
drained = wait_queue_drain(300)
|
|
phase(out, phases, 'sustained', t, injection, restore)
|
|
sustained_after = snapshot(out, 'sustained-after')
|
|
(out / 'sustained.json').write_text(json.dumps({'active': sustained_active, 'after': sustained_after, 'queue_drained': drained}, indent=2) + '\n')
|
|
|
|
print(utc(), 'resolution under normal load', flush=True)
|
|
t = time.time()
|
|
load = run_load(out, 'resolution', args.resolution)
|
|
finish_load(*load, 'resolution', args.resolution)
|
|
phase(out, phases, 'resolution_wait', t)
|
|
snapshot(out, 'resolution')
|
|
|
|
t = time.time()
|
|
compose('stop', 'worker')
|
|
load = run_load(out, 'ttr', args.ttr, ratio=1.0)
|
|
time.sleep(10)
|
|
injection = utc()
|
|
timeline.injection = injection
|
|
print(utc(), 'ttr: worker stopped and uploads started', flush=True)
|
|
time.sleep(args.ttr)
|
|
compose('up', '-d', '--no-deps', 'worker')
|
|
restore = utc()
|
|
finish_load(*load, 'ttr', args.ttr, allow_failures=True)
|
|
wait_queue_drain(300)
|
|
phase(out, phases, 'ttr', t, injection, restore)
|
|
snapshot(out, 'ttr-after')
|
|
|
|
t = time.time()
|
|
load = run_load(out, 'false-positive', args.false_positive)
|
|
print(utc(), 'false-positive phase started', flush=True)
|
|
finish_load(*load, 'false-positive', args.false_positive)
|
|
phase(out, phases, 'false-positive', t)
|
|
final = snapshot(out, 'final')
|
|
(out / 'final.json').write_text(json.dumps(final, indent=2) + '\n')
|
|
|
|
except BaseException as exc:
|
|
interrupted = True
|
|
(out / 'aborted.json').write_text(json.dumps({'aborted_utc': utc(), 'error': repr(exc)}, indent=2) + '\n')
|
|
print(utc(), 'ABORT', repr(exc), flush=True)
|
|
raise
|
|
finally:
|
|
if ACTIVE_LOAD[0] is not None:
|
|
process, log, _ = ACTIVE_LOAD[0]
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait()
|
|
log.close()
|
|
ACTIVE_LOAD[0] = None
|
|
restored = restore_stand()
|
|
if not restored and not interrupted:
|
|
exit_code = 1
|
|
changes = timeline.finish()
|
|
(out / 'alerts.json').write_text(json.dumps(changes, indent=2) + '\n')
|
|
try:
|
|
final_summary = snapshot(out, 'final-summary')
|
|
except Exception as exc:
|
|
final_summary = {'error': str(exc)[:200]}
|
|
(out / 'summary.md').write_text(
|
|
'# Full Track 2 run\n\nPhases:\n```json\n' + json.dumps(phases, indent=2) +
|
|
'\n```\n\nFinal snapshot:\n```json\n' + json.dumps(final_summary, indent=2) +
|
|
'\n```\n\nAborted: ' + str(interrupted) + '\nRestored: ' + str(restored) + '\n')
|
|
print(utc(), 'done; evidence', out, flush=True)
|
|
return exit_code
|
|
|
|
|
|
def wait_queue_drain(timeout=300):
|
|
deadline = time.time() + timeout
|
|
good = 0
|
|
while time.time() < deadline:
|
|
try:
|
|
oldest = prom('max(messenger_queue_oldest_job_age_seconds)')
|
|
depth = prom('max(rabbitmq_queue_messages_ready{queue="attachment.process.v1"})')
|
|
if not oldest or not depth:
|
|
good = 0
|
|
time.sleep(10)
|
|
continue
|
|
oldest_v = float(oldest[0]['value'][1])
|
|
depth_v = float(depth[0]['value'][1])
|
|
if oldest_v < 5 and depth_v < 5:
|
|
good += 1
|
|
if good >= 2:
|
|
return True
|
|
else:
|
|
good = 0
|
|
except Exception:
|
|
good = 0
|
|
time.sleep(10)
|
|
return False
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|