Files
mtusi2026lab5/student-tests/failover_metrics.py
T
2026-10-25 21:43:03 +03:00

149 lines
7.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""lab3, уровень 4: по очереди останавливаем КАЖДУЮ реплику API под нагрузкой и меряем.
make failover-metrics (= python student-tests/failover_metrics.py: stop и kill)
python student-tests/failover_metrics.py stop (только плановая остановка)
Режимы:
stop — docker stop (SIGTERM): плановая остановка. Реплика сначала отвечает 503 на /health/ready,
HAProxy выводит её из балансировки, и только потом она закрывает приём (__main__.py).
kill — docker kill (SIGKILL): авария. Процесс умирает сразу, начатые на нём запросы обрываются,
клиент повторяет их с тем же Idempotency-Key.
Нагрузка (loadgen.py): 4 потока пишут сообщения без пауз, 1 поток читает историю, timeout клиента 2 с.
Меряем: успешные записи в секунду до и после отказа, ошибки и их коды, через сколько HAProxy вывел
реплику (по его статистике, опрос раз в 0,1 с), через сколько прекратились ошибки, отвечала ли
остановленная реплика после остановки, нет ли потерь и дублей. Затем реплика запускается снова
(docker start) и ждём, пока HAProxy вернёт её, — только потом следующий прогон.
Итог: таблица в консоли и evidence/failover-metrics.json (без токенов).
"""
import json
import platform
import sys
import threading
import time
from helpers import ROOT, check, run
from loadgen import CLIENT_TIMEOUT_SECONDS, WRITERS, Load, summary
from stand import api_containers, docker, lb_up_count, wait_balanced
WARMUP_SECONDS = 3
AFTER_SECONDS = 10
OUTPUT = ROOT / "evidence" / "failover-metrics.json"
class LbWatch:
"""Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP)."""
def __init__(self):
self.points, self.stopping = [], threading.Event()
self.thread = threading.Thread(target=self._run, daemon=True)
def _run(self):
while not self.stopping.is_set():
try:
self.points.append((time.monotonic(), lb_up_count()))
except OSError:
pass
time.sleep(0.1)
def start(self):
self.thread.start()
def stop(self):
self.stopping.set()
self.thread.join(timeout=3)
def first_drop(self, after, replicas):
return next((t for t, up in self.points if t >= after and up < replicas), None)
def measure(container, mode, replicas):
wait_balanced(replicas)
load, watch = Load(), LbWatch()
watch.start()
load.start()
time.sleep(WARMUP_SECONDS)
t0 = time.monotonic()
if mode == "stop":
docker("stop", "--time", "8", container)
else:
docker("kill", container)
command_seconds = time.monotonic() - t0
time.sleep(max(0.0, AFTER_SECONDS - command_seconds))
end = t0 + AFTER_SECONDS
load.stop()
watch.stop()
errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]]
errors_end = max((e[0] + e[5] for e in errors), default=None)
drop = watch.first_drop(t0, replicas)
served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]]
target_hits = sum(1 for instance in served_after if instance and container.startswith(instance))
integrity = load.verify()
started = time.monotonic()
docker("start", container)
wait_balanced(replicas)
rejoin_seconds = time.monotonic() - started
return {
"replica": container[:12],
"mode": mode,
"command_seconds": round(command_seconds, 2),
"lb_excluded_after_seconds": round(drop - t0, 2) if drop else None,
"errors": len(errors),
"error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"],
"errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0,
"before": summary(load, t0 - WARMUP_SECONDS, t0),
"after": summary(load, t0, end),
"stopped_replica_answers_after_stop": target_hits,
"integrity": integrity,
"rejoin_seconds": round(rejoin_seconds, 2),
}
def main():
modes = sys.argv[1:] or ["stop", "kill"]
check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]")
containers = api_containers()
replicas = len(containers)
check(replicas >= 2, "need at least 2 running api replicas")
print(f"api replicas: {[c[:12] for c in containers]}; load: {WRITERS} writers without pauses + 1 reader, "
f"client timeout {CLIENT_TIMEOUT_SECONDS} s, retry with the same Idempotency-Key; "
f"{WARMUP_SECONDS} s before and {AFTER_SECONDS} s after the failure")
runs = []
for container in containers:
for mode in modes:
print(f"--- {mode} {container[:12]} ...")
run = measure(container, mode, replicas)
runs.append(run)
print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after "
f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, "
f"last error ended {run['errors_stopped_after_seconds']} s after the failure")
print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; "
f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; "
f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}")
print(f" confirmed {run['integrity']['confirmed_writes']}, in history "
f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; "
f"back in the balancer {run['rejoin_seconds']} s after docker start")
check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered")
check(run["after"]["write_rps"] > 0, "no successful writes after the failure")
print()
print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s")
for r in runs:
print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} "
f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} "
f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}")
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_text(json.dumps({
"profile": {"writers": WRITERS, "readers": 1, "client_timeout_seconds": CLIENT_TIMEOUT_SECONDS,
"warmup_seconds": WARMUP_SECONDS, "after_seconds": AFTER_SECONDS, "replicas": replicas,
"client_python": platform.python_version()},
"runs": runs,
}, indent=2) + "\n", encoding="utf-8")
print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; "
f"saved {OUTPUT.relative_to(ROOT).as_posix()}")
if __name__ == "__main__":
run(main)