forked from mtusi2026/mtusi2026lab3
156 lines
5.4 KiB
Python
156 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def sh(*cmd):
|
|
return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def api_containers():
|
|
out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api')
|
|
return [x[:12] for x in out.split()]
|
|
|
|
|
|
def wait_healthy(count, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}')
|
|
if out.count('(healthy)') >= count:
|
|
return
|
|
time.sleep(1)
|
|
raise Failure('replicas did not become healthy')
|
|
|
|
|
|
class Load:
|
|
def __init__(self, base, ca, concurrency):
|
|
c = Client(base, ca)
|
|
self.alice, _, _, self.chat = c.bootstrap()
|
|
self.base, self.ca, self.n = base, ca, concurrency
|
|
self.events = []
|
|
self.lock = threading.Lock()
|
|
self.stop = threading.Event()
|
|
self.t0 = time.monotonic()
|
|
|
|
def worker(self):
|
|
c = Client(self.base, self.ca, timeout=3)
|
|
while not self.stop.is_set():
|
|
key = uuid.uuid4().hex
|
|
first_error = None
|
|
while not self.stop.is_set():
|
|
try:
|
|
r = c.message(self.alice, self.chat, 'k-' + key, key=key)
|
|
t = time.monotonic() - self.t0
|
|
with self.lock:
|
|
self.events.append((t, True, r['headers']['X-Instance-Id'], first_error))
|
|
break
|
|
except Failure:
|
|
t = time.monotonic() - self.t0
|
|
first_error = first_error or t
|
|
with self.lock:
|
|
self.events.append((t, False, None, None))
|
|
time.sleep(0.05)
|
|
|
|
def start(self):
|
|
self.pool = ThreadPoolExecutor(max_workers=self.n)
|
|
self.futures = [self.pool.submit(self.worker) for _ in range(self.n)]
|
|
|
|
def finish(self):
|
|
self.stop.set()
|
|
self.pool.shutdown(wait=True)
|
|
|
|
def summary(self, begin, end):
|
|
ev = [e for e in self.events if begin <= e[0] < end]
|
|
ok = [e for e in ev if e[1]]
|
|
bad = [e for e in ev if not e[1]]
|
|
span = max(end - begin, 1e-9)
|
|
return {
|
|
'window_seconds': round(span, 2),
|
|
'successful_requests': len(ok),
|
|
'failed_attempts': len(bad),
|
|
'successful_rps': round(len(ok) / span, 1),
|
|
'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None,
|
|
'instances': sorted({e[2] for e in ok}),
|
|
}
|
|
|
|
def instances_between(self, begin, end=float('inf')):
|
|
return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end})
|
|
|
|
|
|
def failover_each(args):
|
|
ids = api_containers()
|
|
check(len(ids) >= 2, 'need at least two replicas')
|
|
load = Load(args.base_url, args.ca_file, args.concurrency)
|
|
load.start()
|
|
time.sleep(args.warmup)
|
|
results = []
|
|
base = load.summary(2, args.warmup)
|
|
for target in ids:
|
|
before = time.monotonic() - load.t0
|
|
sh('docker', 'stop', '-t', '10', target)
|
|
stopped = time.monotonic() - load.t0
|
|
time.sleep(args.hold)
|
|
after_stop = time.monotonic() - load.t0
|
|
window = load.summary(before, after_stop)
|
|
sh('docker', 'start', target)
|
|
wait_healthy(len(ids))
|
|
restored = time.monotonic() - load.t0
|
|
time.sleep(args.warmup)
|
|
results.append({
|
|
'stopped_replica': target,
|
|
'during_failure': window,
|
|
'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop),
|
|
'stop_command_seconds': round(stopped - before, 2),
|
|
'restore_seconds': round(restored - after_stop, 2),
|
|
'after_restore': load.summary(restored + 1, restored + args.warmup),
|
|
})
|
|
load.finish()
|
|
return {'baseline': base, 'replicas': results}
|
|
|
|
|
|
def rolling(args):
|
|
old = api_containers()
|
|
check(len(old) >= 2, 'need at least two replicas')
|
|
n = len(old)
|
|
load = Load(args.base_url, args.ca_file, args.concurrency)
|
|
load.start()
|
|
time.sleep(args.warmup)
|
|
begin = time.monotonic() - load.t0
|
|
sh('scripts/rolling-update.sh')
|
|
end = time.monotonic() - load.t0
|
|
time.sleep(args.warmup)
|
|
load.finish()
|
|
return {
|
|
'old_replicas': old,
|
|
'new_replicas': api_containers(),
|
|
'update_window': load.summary(begin, end),
|
|
'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old],
|
|
}
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True)
|
|
p.add_argument('--concurrency', type=int, default=8)
|
|
p.add_argument('--warmup', type=float, default=8)
|
|
p.add_argument('--hold', type=float, default=12)
|
|
p.add_argument('--output')
|
|
args = arguments('Failure and rolling update under load', configure)
|
|
report = failover_each(args) if args.mode == 'failover-each' else rolling(args)
|
|
text = json.dumps(report, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + '\n')
|
|
print(text)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|