Files
2026-11-06 17:30:30 +03:00

396 lines
16 KiB
Python
Executable File

#!/usr/bin/env python3
"""Full Track 2 timeline with fail-fast, preflight and guaranteed restore.
Warmup >= 70m, baseline, 2m burst, 15m sustained outage, resolution, TTR and
false-positive phases. The script records injection/restore timestamps, alert
transitions and Prometheus snapshots before and after each fault.
"""
import argparse
import csv
import json
import os
import signal
import subprocess
import sys
import threading
import time
import urllib.parse
import urllib.request
from pathlib import Path
for _key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
os.environ.pop(_key, None)
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
os.environ['no_proxy'] = 'localhost,127.0.0.1,::1'
# tests/client.py builds its own opener; make it ignore macOS system proxies too.
urllib.request.getproxies = lambda: {}
OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
ROOT = Path(__file__).resolve().parents[1]
ACTIVE_LOAD = [None]
def utc(ts=None):
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(ts if ts is not None else time.time()))
def http_json(url, timeout=15):
with OPENER.open(url, timeout=timeout) as response:
return json.loads(response.read())
def prom(expr):
url = 'http://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
return http_json(url).get('data', {}).get('result', [])
def alerts():
return http_json('http://localhost:9090/api/v1/alerts').get('data', {}).get('alerts', [])
def compose(*args, check=True):
result = subprocess.run(
['docker', 'compose', '--env-file', '.env', '-f', 'compose.yaml', *args],
cwd=ROOT, capture_output=True, text=True, check=False,
)
(ROOT / 'evidence/lab6/track2').mkdir(parents=True, exist_ok=True)
with (ROOT / 'evidence/lab6/track2/full-compose.log').open('a') as log:
log.write(utc() + ' docker compose ' + ' '.join(args) + ' rc=' + str(result.returncode) + '\n')
if result.stderr:
log.write(result.stderr[-2000:] + '\n')
if check and result.returncode != 0:
raise RuntimeError('docker compose %s failed rc=%d: %s' % (' '.join(args), result.returncode, result.stderr[-500:]))
return result
def wait_ready(timeout=120):
deadline = time.time() + timeout
while time.time() < deadline:
try:
with OPENER.open('http://localhost:8080/health/ready', timeout=5) as response:
if response.status == 200:
return True
except Exception:
pass
time.sleep(2)
return False
class AlertTimeline:
def __init__(self, path):
self.path = path
self.stop = threading.Event()
self.active = {}
self.changes = []
self.injection = ''
self.lock = threading.Lock()
path.parent.mkdir(parents=True, exist_ok=True)
self.handle = path.open('w', newline='')
self.writer = csv.writer(self.handle)
self.writer.writerow(['observed_utc', 'alert', 'state', 'active_at', 'value', 'injection_utc', 'labels'])
self.thread = threading.Thread(target=self.run, daemon=True)
def start(self):
self.thread.start()
def run(self):
while not self.stop.is_set():
try:
current = {}
for alert in alerts():
state = alert.get('state', '')
if state not in ('pending', 'firing'):
continue
labels = alert.get('labels', {})
name = labels.get('alertname', '')
key = name + '|' + json.dumps(labels, sort_keys=True, separators=(',', ':'))
current[key] = (name, state, alert.get('activeAt', ''), alert.get('value', ''), labels)
for key, (name, state, active_at, value, labels) in current.items():
previous = self.active.get(key)
if previous is None or previous[1] != state:
row = [utc(), name, state, active_at, value, self.injection, json.dumps(labels, sort_keys=True, separators=(',', ':'))]
with self.lock:
self.changes.append(row)
self.writer.writerow(row)
self.handle.flush()
for key in sorted(set(self.active) - set(current)):
name, state, _, _, labels = self.active[key]
if state != 'firing':
continue
row = [utc(), name, 'resolved', '', '', self.injection, json.dumps(labels, sort_keys=True, separators=(',', ':'))]
with self.lock:
self.changes.append(row)
self.writer.writerow(row)
self.handle.flush()
self.active = current
except Exception as exc:
with self.lock:
self.writer.writerow([utc(), 'POLL_ERROR', str(exc)[:160], '', '', self.injection])
self.handle.flush()
self.stop.wait(5)
def finish(self):
self.stop.set()
self.thread.join(timeout=20)
self.handle.close()
return self.changes
def snapshot(out, label):
values = {}
errors = []
for name, expr in {
'api_error_5m': 'slo:api_error_ratio:5m',
'api_error_1h': 'slo:api_error_ratio:1h',
'api_error_30m': 'slo:api_error_ratio:30m',
'api_error_6h': 'slo:api_error_ratio:6h',
'ttr_bad_5m': 'slo:job_ttr_bad_ratio:5m',
'ttr_bad_1h': 'slo:job_ttr_bad_ratio:1h',
'queue_oldest': 'max(messenger_queue_oldest_job_age_seconds)',
'queue_depth': 'max(rabbitmq_queue_messages_ready{queue="attachment.process.v1"})',
'jobs_overdue': 'max(messenger_jobs_overdue)',
'worker_busy': 'max(messenger_worker_busy)',
}.items():
try:
result = prom(expr)
values[name] = result[0]['value'][1] if result else None
except Exception as exc:
values[name] = None
errors.append(name + ': ' + str(exc)[:100])
(out / 'snapshots').mkdir(parents=True, exist_ok=True)
(out / 'snapshots' / (label + '-' + utc().replace(':', '-') + '.json')).write_text(json.dumps(values, indent=2) + '\n')
if errors:
raise RuntimeError('Prometheus snapshot failed: ' + '; '.join(errors))
return values
def run_load(out, label, duration, ratio=0.2, rate=0.2):
log_path = out / (label + '-load.log')
output_path = out / (label + '-load.json')
cmd = [sys.executable, str(ROOT / 'scripts/lab6_load.py'),
'--base-url', 'http://localhost:8080', '--rate', str(rate),
'--duration', str(duration), '--upload-ratio', str(ratio),
'--deadline', '60', '--output', str(output_path)]
log = log_path.open('w')
process = subprocess.Popen(cmd, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT)
ACTIVE_LOAD[0] = (process, log, output_path)
return process, log, output_path
def finish_load(process, log, output_path, label, expected_s, allow_failures=False):
try:
rc = process.wait(timeout=expected_s + 300)
except subprocess.TimeoutExpired:
process.kill()
rc = process.wait()
log.close()
ACTIVE_LOAD[0] = None
if not output_path.exists():
raise RuntimeError(label + ': load produced no JSON (rc=' + str(rc) + ')')
summary = json.loads(output_path.read_text())
minimum = 0.85 * expected_s if expected_s >= 30 else 0.5 * expected_s
if summary.get('duration_s', 0) < minimum:
raise RuntimeError(label + ': load ran only ' + str(summary.get('duration_s')) + 's, expected ~' + str(expected_s))
if not allow_failures and (rc != 0 or summary.get('failures', 0)):
raise RuntimeError(label + ': load failed rc=' + str(rc) + ' failures=' + str(summary.get('failures')))
return summary
def restore_stand():
compose('unpause', 'postgres', check=False)
compose('start', 'postgres', check=False)
compose('up', '-d', '--wait', '--no-deps', 'worker', check=False)
return wait_ready(120)
def phase(out, phases, name, started, injection='', restore=''):
value = {
'phase': name,
'started_utc': utc(started),
'finished_utc': utc(),
'duration_s': round(time.time() - started, 1),
'injection_utc': injection,
'restore_utc': restore,
}
phases.append(value)
(out / 'phases.json').write_text(json.dumps(phases, indent=2) + '\n')
def main():
parser = argparse.ArgumentParser(description='Full lab6 Track 2 timeline')
parser.add_argument('--warmup', type=int, default=4200)
parser.add_argument('--baseline', type=int, default=1200)
parser.add_argument('--burst', type=int, default=120)
parser.add_argument('--settle', type=int, default=600)
parser.add_argument('--sustained', type=int, default=900)
parser.add_argument('--resolution', type=int, default=3600)
parser.add_argument('--ttr', type=int, default=900)
parser.add_argument('--false-positive', type=int, default=1200)
parser.add_argument('--out', default='evidence/lab6/track2/full')
args = parser.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
phases = []
timeline = AlertTimeline(out / 'alerts.csv')
timeline.start()
interrupted = False
exit_code = 0
try:
def on_signal(signum, _frame):
raise KeyboardInterrupt('signal %d' % signum)
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
if not wait_ready(120):
raise RuntimeError('preflight: /health/ready is not ready')
preflight = run_load(out, 'preflight', 3, ratio=0.5, rate=1.0)
finish_load(*preflight, 'preflight', 3, allow_failures=False)
print(utc(), 'preflight OK; start full track', flush=True)
t = time.time()
load = run_load(out, 'warmup', args.warmup)
print(utc(), 'warmup started', flush=True)
finish_load(*load, 'warmup', args.warmup)
phase(out, phases, 'warmup', t)
snapshot(out, 'warmup')
t = time.time()
load = run_load(out, 'baseline', args.baseline)
print(utc(), 'baseline started', flush=True)
finish_load(*load, 'baseline', args.baseline)
phase(out, phases, 'baseline', t)
(out / 'baseline.json').write_text(json.dumps(snapshot(out, 'baseline'), indent=2) + '\n')
t = time.time()
load = run_load(out, 'burst', args.burst + args.settle)
time.sleep(10)
injection = utc()
timeline.injection = injection
print(utc(), 'burst: pause postgres', flush=True)
compose('pause', 'postgres')
time.sleep(min(30, args.burst))
burst_active = snapshot(out, 'burst-active')
if args.burst > 30:
time.sleep(args.burst - 30)
compose('unpause', 'postgres')
restore = utc()
print(utc(), 'burst: unpause postgres', flush=True)
finish_load(*load, 'burst', args.burst + args.settle, allow_failures=True)
phase(out, phases, 'burst', t, injection, restore)
burst_after = snapshot(out, 'burst-after')
(out / 'burst.json').write_text(json.dumps({'active': burst_active, 'after': burst_after}, indent=2) + '\n')
t = time.time()
load = run_load(out, 'sustained', args.sustained + 300)
time.sleep(10)
injection = utc()
timeline.injection = injection
print(utc(), 'sustained: stop postgres', flush=True)
compose('stop', 'postgres')
time.sleep(min(30, args.sustained))
sustained_active = snapshot(out, 'sustained-active')
if args.sustained > 30:
time.sleep(args.sustained - 30)
compose('start', 'postgres')
if not wait_ready(180):
raise RuntimeError('sustained: readiness did not recover')
restore = utc()
print(utc(), 'sustained: postgres restored', flush=True)
finish_load(*load, 'sustained', args.sustained + 300, allow_failures=True)
drained = wait_queue_drain(300)
phase(out, phases, 'sustained', t, injection, restore)
sustained_after = snapshot(out, 'sustained-after')
(out / 'sustained.json').write_text(json.dumps({'active': sustained_active, 'after': sustained_after, 'queue_drained': drained}, indent=2) + '\n')
print(utc(), 'resolution under normal load', flush=True)
t = time.time()
load = run_load(out, 'resolution', args.resolution)
finish_load(*load, 'resolution', args.resolution)
phase(out, phases, 'resolution_wait', t)
snapshot(out, 'resolution')
t = time.time()
compose('stop', 'worker')
load = run_load(out, 'ttr', args.ttr, ratio=1.0)
time.sleep(10)
injection = utc()
timeline.injection = injection
print(utc(), 'ttr: worker stopped and uploads started', flush=True)
time.sleep(args.ttr)
compose('up', '-d', '--no-deps', 'worker')
restore = utc()
finish_load(*load, 'ttr', args.ttr, allow_failures=True)
wait_queue_drain(300)
phase(out, phases, 'ttr', t, injection, restore)
snapshot(out, 'ttr-after')
t = time.time()
load = run_load(out, 'false-positive', args.false_positive)
print(utc(), 'false-positive phase started', flush=True)
finish_load(*load, 'false-positive', args.false_positive)
phase(out, phases, 'false-positive', t)
final = snapshot(out, 'final')
(out / 'final.json').write_text(json.dumps(final, indent=2) + '\n')
except BaseException as exc:
interrupted = True
(out / 'aborted.json').write_text(json.dumps({'aborted_utc': utc(), 'error': repr(exc)}, indent=2) + '\n')
print(utc(), 'ABORT', repr(exc), flush=True)
raise
finally:
if ACTIVE_LOAD[0] is not None:
process, log, _ = ACTIVE_LOAD[0]
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
log.close()
ACTIVE_LOAD[0] = None
restored = restore_stand()
if not restored and not interrupted:
exit_code = 1
changes = timeline.finish()
(out / 'alerts.json').write_text(json.dumps(changes, indent=2) + '\n')
try:
final_summary = snapshot(out, 'final-summary')
except Exception as exc:
final_summary = {'error': str(exc)[:200]}
(out / 'summary.md').write_text(
'# Full Track 2 run\n\nPhases:\n```json\n' + json.dumps(phases, indent=2) +
'\n```\n\nFinal snapshot:\n```json\n' + json.dumps(final_summary, indent=2) +
'\n```\n\nAborted: ' + str(interrupted) + '\nRestored: ' + str(restored) + '\n')
print(utc(), 'done; evidence', out, flush=True)
return exit_code
def wait_queue_drain(timeout=300):
deadline = time.time() + timeout
good = 0
while time.time() < deadline:
try:
oldest = prom('max(messenger_queue_oldest_job_age_seconds)')
depth = prom('max(rabbitmq_queue_messages_ready{queue="attachment.process.v1"})')
if not oldest or not depth:
good = 0
time.sleep(10)
continue
oldest_v = float(oldest[0]['value'][1])
depth_v = float(depth[0]['value'][1])
if oldest_v < 5 and depth_v < 5:
good += 1
if good >= 2:
return True
else:
good = 0
except Exception:
good = 0
time.sleep(10)
return False
if __name__ == '__main__':
raise SystemExit(main())