Files
mtusi2026lab6-fork/student-tests/failover_series.py
T

164 lines
8.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Уровень 4: последовательная остановка КАЖДОЙ API-реплики под нагрузкой.
Для каждой реплики (по X-Instance-Id): нагрузка -> stop-one -> 15 с работы без неё
-> restore -> возврат реплики в ротацию. Собираются RPS, ошибки, задержки по
окнам, время обнаружения отказа балансировщиком (по странице HAProxy) и время
восстановления. Перед следующей репликой предыдущая уже восстановлена.
python student-tests/failover_series.py --output evidence/failover_series.json
python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json
Остановка/восстановление по умолчанию — scripts/ops.py (docker compose);
для другого стенда их можно переопределить --stop-cmd / --restore-cmd.
"""
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import common # noqa: E402
from common import request # noqa: E402
from loadgen import Load # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
def observe_instances(token, n=40):
seen = set()
for _ in range(n):
r = request("GET", "/api/v1/users/me", token=token, timeout=3)
seen.add(r.headers.get("X-Instance-Id"))
return sorted(x for x in seen if x)
def run_hook(cmd, env_extra, timeout):
env = dict(os.environ, **env_extra)
started = time.monotonic()
proc = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, timeout=timeout)
return proc.returncode, time.monotonic() - started, proc.stdout[-2000:]
def one_round(args, target):
load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start()
time.sleep(args.pre)
up_before = load.lb[-1][1] if load.lb else ()
t_stop = load.now()
rc, stop_s, out = run_hook(args.stop_cmd, {"TARGET_INSTANCE_ID": target, "STOP_MODE": args.mode}, 60)
if rc != 0:
load.stop()
raise SystemExit("stop hook failed for %s:\n%s" % (target, out))
time.sleep(max(0.0, args.post - stop_s))
t_restore = load.now()
rc, restore_s, out = run_hook(args.restore_cmd, {}, 180)
if rc != 0:
load.stop()
raise SystemExit("restore hook failed:\n%s" % out)
# ждём, пока восстановленная реплика снова обслужит запрос
back = None
deadline = time.monotonic() + 30
while back is None and time.monotonic() < deadline:
for e in reversed(load.events):
if e[0] >= t_restore and e[2] and e[4] == target:
back = e[0]
break
time.sleep(0.2)
time.sleep(args.tail)
load.stop()
t_end = load.now()
detect = next((t - t_stop for t, up in load.lb if t >= t_stop and len(up) < len(up_before)), None)
lb_back = next((t - t_restore for t, up in load.lb if t >= t_restore and len(up) >= len(up_before)), None)
errs = load.errors_between(t_stop, t_restore)
# Сколько ещё после SIGTERM реплика отвечала (фаза осушения), до restore.
served_after = [e[0] - t_stop for e in load.events if t_stop <= e[0] < t_restore and e[2] and e[4] == target]
# После возврата hook (реплика остановлена) и до restore она не должна отвечать.
after_stop_ok = [e for e in load.events if t_stop + stop_s <= e[0] < t_restore and e[2]]
return {
"target_instance": target,
"mode": args.mode,
"lb_up_before": list(up_before),
"stop_hook_seconds": round(stop_s, 2),
"lb_detect_down_seconds": round(detect, 2) if detect is not None else None,
"target_last_served_after_stop_seconds": round(max(served_after), 2) if served_after else None,
"target_served_after_hook_returned": any(e[4] == target for e in after_stop_ok),
"errors_during_outage": len(errs),
"error_kinds": sorted({str(e[3]) for e in errs}),
"error_window_seconds": round(max(e[0] for e in errs) - t_stop, 2) if errs else 0.0,
"restore_hook_seconds": round(restore_s, 2),
"lb_back_up_seconds": round(lb_back, 2) if lb_back is not None else None,
"target_back_in_rotation_seconds": round(back - t_restore, 2) if back is not None else None,
"windows": {
"before": load.window(0, t_stop),
"outage_first_10s": load.window(t_stop, t_stop + 10),
"outage_rest": load.window(t_stop + 10, t_restore),
"after_restore": load.window(t_restore, t_end),
},
"lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb],
"data": load.verify(),
}
def main():
py = sys.executable or "python"
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080"))
p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404"))
p.add_argument("--stop-cmd", default=None, help="default: python scripts/ops.py stop-one")
p.add_argument("--restore-cmd", default=None, help="default: python scripts/ops.py restore")
p.add_argument("--mode", choices=["stop", "kill"], default="stop", help="stop = SIGTERM (graceful), kill = SIGKILL")
p.add_argument("--writers", type=int, default=4)
p.add_argument("--readers", type=int, default=2)
p.add_argument("--pre", type=float, default=5)
p.add_argument("--post", type=float, default=15)
p.add_argument("--tail", type=float, default=3)
p.add_argument("--output")
args = p.parse_args()
args.stop_cmd = shlex.split(args.stop_cmd) if args.stop_cmd else [py, "scripts/ops.py", "stop-one"]
args.restore_cmd = shlex.split(args.restore_cmd) if args.restore_cmd else [py, "scripts/ops.py", "restore"]
common.BASE_URL = args.base_url.rstrip("/")
probe = common.actor("probe")
instances = observe_instances(probe["_token"])
if len(instances) < 2:
raise SystemExit("need >= 2 replicas behind LB, observed %s" % instances)
print("replicas observed through LB: %s" % instances, flush=True)
rounds = []
for target in instances:
print("-- stopping %s (mode=%s) under load" % (target, args.mode), flush=True)
r = one_round(args, target)
rounds.append(r)
w = r["windows"]
print(" hook %.2fs | LB DOWN after %s s | errors %d (%s) | RPS before %.1f, outage %.1f, after restore %.1f"
" | p95 outage %s ms | back in rotation %s s | data ok=%s"
% (r["stop_hook_seconds"], r["lb_detect_down_seconds"], r["errors_during_outage"],
",".join(r["error_kinds"]) or "-", w["before"]["ok_rps"], w["outage_first_10s"]["ok_rps"],
w["after_restore"]["ok_rps"], w["outage_first_10s"]["latency_ms_ok"]["p95"],
r["target_back_in_rotation_seconds"], r["data"]["ok"]), flush=True)
report = {
"scenario": "stop each API replica in turn under load, restore before the next",
"load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0,
"write_retry": "same Idempotency-Key and body until 201"},
"replicas": instances,
"rounds": rounds,
}
ok = all(r["data"]["ok"] and r["target_back_in_rotation_seconds"] is not None
and not r["target_served_after_hook_returned"] for r in rounds)
report["pass"] = ok
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(text + "\n", encoding="utf-8")
print(("PASS" if ok else "FAIL") + ": every replica stopped and restored; confirmed data intact, no duplicates")
sys.exit(0 if ok else 1)
if __name__ == "__main__":
main()