forked from mtusi2026/mtusi2026lab3
130 lines
5.5 KiB
Python
130 lines
5.5 KiB
Python
import json
|
||
import platform
|
||
import sys
|
||
import threading
|
||
import time
|
||
|
||
from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced
|
||
from loadgen import Load, summary
|
||
|
||
WARMUP_SECONDS = 3
|
||
AFTER_SECONDS = 10
|
||
OUTPUT = ROOT / "evidence" / "failover-metrics.json"
|
||
|
||
|
||
class LbWatch:
|
||
"""Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP)."""
|
||
|
||
def __init__(self):
|
||
self.points, self.stopping = [], threading.Event()
|
||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||
|
||
def _run(self):
|
||
while not self.stopping.is_set():
|
||
try:
|
||
self.points.append((time.monotonic(), lb_up_count()))
|
||
except OSError:
|
||
pass
|
||
time.sleep(0.1)
|
||
|
||
def start(self):
|
||
self.thread.start()
|
||
|
||
def stop(self):
|
||
self.stopping.set()
|
||
self.thread.join(timeout=3)
|
||
|
||
def first_drop(self, after, replicas):
|
||
return next((t for t, up in self.points if t >= after and up < replicas), None)
|
||
|
||
|
||
def measure(container, mode, replicas):
|
||
wait_balanced(replicas)
|
||
load, watch = Load(), LbWatch()
|
||
watch.start()
|
||
load.start()
|
||
time.sleep(WARMUP_SECONDS)
|
||
t0 = time.monotonic()
|
||
if mode == "stop":
|
||
docker("stop", "--time", "8", container)
|
||
else:
|
||
docker("kill", container)
|
||
command_seconds = time.monotonic() - t0
|
||
time.sleep(max(0.0, AFTER_SECONDS - command_seconds))
|
||
end = t0 + AFTER_SECONDS
|
||
load.stop()
|
||
watch.stop()
|
||
|
||
errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]]
|
||
errors_end = max((e[0] + e[5] for e in errors), default=None)
|
||
drop = watch.first_drop(t0, replicas)
|
||
served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]]
|
||
target_hits = sum(1 for instance in served_after if instance and container.startswith(instance))
|
||
integrity = load.verify()
|
||
|
||
started = time.monotonic()
|
||
docker("start", container)
|
||
wait_balanced(replicas)
|
||
rejoin_seconds = time.monotonic() - started
|
||
|
||
return {
|
||
"replica": container[:12],
|
||
"mode": mode,
|
||
"command_seconds": round(command_seconds, 2),
|
||
"lb_excluded_after_seconds": round(drop - t0, 2) if drop else None,
|
||
"errors": len(errors),
|
||
"error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"],
|
||
"errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0,
|
||
"before": summary(load, t0 - WARMUP_SECONDS, t0),
|
||
"after": summary(load, t0, end),
|
||
"stopped_replica_answers_after_stop": target_hits,
|
||
"integrity": integrity,
|
||
"rejoin_seconds": round(rejoin_seconds, 2),
|
||
}
|
||
|
||
|
||
def main():
|
||
modes = sys.argv[1:] or ["stop", "kill"]
|
||
check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]")
|
||
containers = api_containers()
|
||
replicas = len(containers)
|
||
check(replicas >= 2, "need at least 2 running api replicas")
|
||
print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, "
|
||
f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and "
|
||
f"{AFTER_SECONDS} s after the failure")
|
||
runs = []
|
||
for container in containers:
|
||
for mode in modes:
|
||
print(f"--- {mode} {container[:12]} ...")
|
||
run = measure(container, mode, replicas)
|
||
runs.append(run)
|
||
print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after "
|
||
f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, "
|
||
f"stopped after {run['errors_stopped_after_seconds']} s")
|
||
print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; "
|
||
f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; "
|
||
f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}")
|
||
print(f" confirmed {run['integrity']['confirmed_writes']}, in history "
|
||
f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; "
|
||
f"back in the balancer {run['rejoin_seconds']} s after docker start")
|
||
check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered")
|
||
check(run["after"]["write_rps"] > 0, "no successful writes after the failure")
|
||
print()
|
||
print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s")
|
||
for r in runs:
|
||
print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} "
|
||
f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} "
|
||
f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}")
|
||
OUTPUT.parent.mkdir(exist_ok=True)
|
||
OUTPUT.write_text(json.dumps({
|
||
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS,
|
||
"after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()},
|
||
"runs": runs,
|
||
}, indent=2) + "\n", encoding="utf-8")
|
||
print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; "
|
||
f"saved {OUTPUT.relative_to(ROOT)}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|