Files
mtusi2026lab5-fork/tests/scenario.py
T

156 lines
5.4 KiB
Python

#!/usr/bin/env python3
from concurrent.futures import ThreadPoolExecutor
import json
import subprocess
import threading
import time
import uuid
from pathlib import Path
from client import Client, Failure, arguments, check, run
ROOT = Path(__file__).resolve().parent.parent
def sh(*cmd):
return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
def api_containers():
out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api')
return [x[:12] for x in out.split()]
def wait_healthy(count, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}')
if out.count('(healthy)') >= count:
return
time.sleep(1)
raise Failure('replicas did not become healthy')
class Load:
def __init__(self, base, ca, concurrency):
c = Client(base, ca)
self.alice, _, _, self.chat = c.bootstrap()
self.base, self.ca, self.n = base, ca, concurrency
self.events = []
self.lock = threading.Lock()
self.stop = threading.Event()
self.t0 = time.monotonic()
def worker(self):
c = Client(self.base, self.ca, timeout=3)
while not self.stop.is_set():
key = uuid.uuid4().hex
first_error = None
while not self.stop.is_set():
try:
r = c.message(self.alice, self.chat, 'k-' + key, key=key)
t = time.monotonic() - self.t0
with self.lock:
self.events.append((t, True, r['headers']['X-Instance-Id'], first_error))
break
except Failure:
t = time.monotonic() - self.t0
first_error = first_error or t
with self.lock:
self.events.append((t, False, None, None))
time.sleep(0.05)
def start(self):
self.pool = ThreadPoolExecutor(max_workers=self.n)
self.futures = [self.pool.submit(self.worker) for _ in range(self.n)]
def finish(self):
self.stop.set()
self.pool.shutdown(wait=True)
def summary(self, begin, end):
ev = [e for e in self.events if begin <= e[0] < end]
ok = [e for e in ev if e[1]]
bad = [e for e in ev if not e[1]]
span = max(end - begin, 1e-9)
return {
'window_seconds': round(span, 2),
'successful_requests': len(ok),
'failed_attempts': len(bad),
'successful_rps': round(len(ok) / span, 1),
'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None,
'instances': sorted({e[2] for e in ok}),
}
def instances_between(self, begin, end=float('inf')):
return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end})
def failover_each(args):
ids = api_containers()
check(len(ids) >= 2, 'need at least two replicas')
load = Load(args.base_url, args.ca_file, args.concurrency)
load.start()
time.sleep(args.warmup)
results = []
base = load.summary(2, args.warmup)
for target in ids:
before = time.monotonic() - load.t0
sh('docker', 'stop', '-t', '10', target)
stopped = time.monotonic() - load.t0
time.sleep(args.hold)
after_stop = time.monotonic() - load.t0
window = load.summary(before, after_stop)
sh('docker', 'start', target)
wait_healthy(len(ids))
restored = time.monotonic() - load.t0
time.sleep(args.warmup)
results.append({
'stopped_replica': target,
'during_failure': window,
'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop),
'stop_command_seconds': round(stopped - before, 2),
'restore_seconds': round(restored - after_stop, 2),
'after_restore': load.summary(restored + 1, restored + args.warmup),
})
load.finish()
return {'baseline': base, 'replicas': results}
def rolling(args):
old = api_containers()
check(len(old) >= 2, 'need at least two replicas')
n = len(old)
load = Load(args.base_url, args.ca_file, args.concurrency)
load.start()
time.sleep(args.warmup)
begin = time.monotonic() - load.t0
sh('scripts/rolling-update.sh')
end = time.monotonic() - load.t0
time.sleep(args.warmup)
load.finish()
return {
'old_replicas': old,
'new_replicas': api_containers(),
'update_window': load.summary(begin, end),
'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old],
}
def main():
def configure(p):
p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True)
p.add_argument('--concurrency', type=int, default=8)
p.add_argument('--warmup', type=float, default=8)
p.add_argument('--hold', type=float, default=12)
p.add_argument('--output')
args = arguments('Failure and rolling update under load', configure)
report = failover_each(args) if args.mode == 'failover-each' else rolling(args)
text = json.dumps(report, indent=2)
if args.output:
Path(args.output).write_text(text + '\n')
print(text)
if __name__ == '__main__':
run(main)