Files

130 lines
5.5 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import platform
import sys
import threading
import time
from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced
from loadgen import Load, summary
WARMUP_SECONDS = 3
AFTER_SECONDS = 10
OUTPUT = ROOT / "evidence" / "failover-metrics.json"
class LbWatch:
"""Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP)."""
def __init__(self):
self.points, self.stopping = [], threading.Event()
self.thread = threading.Thread(target=self._run, daemon=True)
def _run(self):
while not self.stopping.is_set():
try:
self.points.append((time.monotonic(), lb_up_count()))
except OSError:
pass
time.sleep(0.1)
def start(self):
self.thread.start()
def stop(self):
self.stopping.set()
self.thread.join(timeout=3)
def first_drop(self, after, replicas):
return next((t for t, up in self.points if t >= after and up < replicas), None)
def measure(container, mode, replicas):
wait_balanced(replicas)
load, watch = Load(), LbWatch()
watch.start()
load.start()
time.sleep(WARMUP_SECONDS)
t0 = time.monotonic()
if mode == "stop":
docker("stop", "--time", "8", container)
else:
docker("kill", container)
command_seconds = time.monotonic() - t0
time.sleep(max(0.0, AFTER_SECONDS - command_seconds))
end = t0 + AFTER_SECONDS
load.stop()
watch.stop()
errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]]
errors_end = max((e[0] + e[5] for e in errors), default=None)
drop = watch.first_drop(t0, replicas)
served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]]
target_hits = sum(1 for instance in served_after if instance and container.startswith(instance))
integrity = load.verify()
started = time.monotonic()
docker("start", container)
wait_balanced(replicas)
rejoin_seconds = time.monotonic() - started
return {
"replica": container[:12],
"mode": mode,
"command_seconds": round(command_seconds, 2),
"lb_excluded_after_seconds": round(drop - t0, 2) if drop else None,
"errors": len(errors),
"error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"],
"errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0,
"before": summary(load, t0 - WARMUP_SECONDS, t0),
"after": summary(load, t0, end),
"stopped_replica_answers_after_stop": target_hits,
"integrity": integrity,
"rejoin_seconds": round(rejoin_seconds, 2),
}
def main():
modes = sys.argv[1:] or ["stop", "kill"]
check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]")
containers = api_containers()
replicas = len(containers)
check(replicas >= 2, "need at least 2 running api replicas")
print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, "
f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and "
f"{AFTER_SECONDS} s after the failure")
runs = []
for container in containers:
for mode in modes:
print(f"--- {mode} {container[:12]} ...")
run = measure(container, mode, replicas)
runs.append(run)
print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after "
f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, "
f"stopped after {run['errors_stopped_after_seconds']} s")
print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; "
f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; "
f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}")
print(f" confirmed {run['integrity']['confirmed_writes']}, in history "
f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; "
f"back in the balancer {run['rejoin_seconds']} s after docker start")
check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered")
check(run["after"]["write_rps"] > 0, "no successful writes after the failure")
print()
print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s")
for r in runs:
print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} "
f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} "
f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}")
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_text(json.dumps({
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS,
"after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()},
"runs": runs,
}, indent=2) + "\n", encoding="utf-8")
print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; "
f"saved {OUTPUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()