forked from mtusi2026/mtusi2026lab6
164 lines
8.1 KiB
Python
164 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
||
"""Уровень 4: последовательная остановка КАЖДОЙ API-реплики под нагрузкой.
|
||
|
||
Для каждой реплики (по X-Instance-Id): нагрузка -> stop-one -> 15 с работы без неё
|
||
-> restore -> возврат реплики в ротацию. Собираются RPS, ошибки, задержки по
|
||
окнам, время обнаружения отказа балансировщиком (по странице HAProxy) и время
|
||
восстановления. Перед следующей репликой предыдущая уже восстановлена.
|
||
|
||
python student-tests/failover_series.py --output evidence/failover_series.json
|
||
python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json
|
||
|
||
Остановка/восстановление по умолчанию — scripts/ops.py (docker compose);
|
||
для другого стенда их можно переопределить --stop-cmd / --restore-cmd.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
import common # noqa: E402
|
||
from common import request # noqa: E402
|
||
from loadgen import Load # noqa: E402
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
|
||
|
||
def observe_instances(token, n=40):
|
||
seen = set()
|
||
for _ in range(n):
|
||
r = request("GET", "/api/v1/users/me", token=token, timeout=3)
|
||
seen.add(r.headers.get("X-Instance-Id"))
|
||
return sorted(x for x in seen if x)
|
||
|
||
|
||
def run_hook(cmd, env_extra, timeout):
|
||
env = dict(os.environ, **env_extra)
|
||
started = time.monotonic()
|
||
proc = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||
text=True, timeout=timeout)
|
||
return proc.returncode, time.monotonic() - started, proc.stdout[-2000:]
|
||
|
||
|
||
def one_round(args, target):
|
||
load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start()
|
||
time.sleep(args.pre)
|
||
up_before = load.lb[-1][1] if load.lb else ()
|
||
t_stop = load.now()
|
||
rc, stop_s, out = run_hook(args.stop_cmd, {"TARGET_INSTANCE_ID": target, "STOP_MODE": args.mode}, 60)
|
||
if rc != 0:
|
||
load.stop()
|
||
raise SystemExit("stop hook failed for %s:\n%s" % (target, out))
|
||
time.sleep(max(0.0, args.post - stop_s))
|
||
t_restore = load.now()
|
||
rc, restore_s, out = run_hook(args.restore_cmd, {}, 180)
|
||
if rc != 0:
|
||
load.stop()
|
||
raise SystemExit("restore hook failed:\n%s" % out)
|
||
# ждём, пока восстановленная реплика снова обслужит запрос
|
||
back = None
|
||
deadline = time.monotonic() + 30
|
||
while back is None and time.monotonic() < deadline:
|
||
for e in reversed(load.events):
|
||
if e[0] >= t_restore and e[2] and e[4] == target:
|
||
back = e[0]
|
||
break
|
||
time.sleep(0.2)
|
||
time.sleep(args.tail)
|
||
load.stop()
|
||
t_end = load.now()
|
||
|
||
detect = next((t - t_stop for t, up in load.lb if t >= t_stop and len(up) < len(up_before)), None)
|
||
lb_back = next((t - t_restore for t, up in load.lb if t >= t_restore and len(up) >= len(up_before)), None)
|
||
errs = load.errors_between(t_stop, t_restore)
|
||
# Сколько ещё после SIGTERM реплика отвечала (фаза осушения), до restore.
|
||
served_after = [e[0] - t_stop for e in load.events if t_stop <= e[0] < t_restore and e[2] and e[4] == target]
|
||
# После возврата hook (реплика остановлена) и до restore она не должна отвечать.
|
||
after_stop_ok = [e for e in load.events if t_stop + stop_s <= e[0] < t_restore and e[2]]
|
||
return {
|
||
"target_instance": target,
|
||
"mode": args.mode,
|
||
"lb_up_before": list(up_before),
|
||
"stop_hook_seconds": round(stop_s, 2),
|
||
"lb_detect_down_seconds": round(detect, 2) if detect is not None else None,
|
||
"target_last_served_after_stop_seconds": round(max(served_after), 2) if served_after else None,
|
||
"target_served_after_hook_returned": any(e[4] == target for e in after_stop_ok),
|
||
"errors_during_outage": len(errs),
|
||
"error_kinds": sorted({str(e[3]) for e in errs}),
|
||
"error_window_seconds": round(max(e[0] for e in errs) - t_stop, 2) if errs else 0.0,
|
||
"restore_hook_seconds": round(restore_s, 2),
|
||
"lb_back_up_seconds": round(lb_back, 2) if lb_back is not None else None,
|
||
"target_back_in_rotation_seconds": round(back - t_restore, 2) if back is not None else None,
|
||
"windows": {
|
||
"before": load.window(0, t_stop),
|
||
"outage_first_10s": load.window(t_stop, t_stop + 10),
|
||
"outage_rest": load.window(t_stop + 10, t_restore),
|
||
"after_restore": load.window(t_restore, t_end),
|
||
},
|
||
"lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb],
|
||
"data": load.verify(),
|
||
}
|
||
|
||
|
||
def main():
|
||
py = sys.executable or "python"
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080"))
|
||
p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404"))
|
||
p.add_argument("--stop-cmd", default=None, help="default: python scripts/ops.py stop-one")
|
||
p.add_argument("--restore-cmd", default=None, help="default: python scripts/ops.py restore")
|
||
p.add_argument("--mode", choices=["stop", "kill"], default="stop", help="stop = SIGTERM (graceful), kill = SIGKILL")
|
||
p.add_argument("--writers", type=int, default=4)
|
||
p.add_argument("--readers", type=int, default=2)
|
||
p.add_argument("--pre", type=float, default=5)
|
||
p.add_argument("--post", type=float, default=15)
|
||
p.add_argument("--tail", type=float, default=3)
|
||
p.add_argument("--output")
|
||
args = p.parse_args()
|
||
args.stop_cmd = shlex.split(args.stop_cmd) if args.stop_cmd else [py, "scripts/ops.py", "stop-one"]
|
||
args.restore_cmd = shlex.split(args.restore_cmd) if args.restore_cmd else [py, "scripts/ops.py", "restore"]
|
||
common.BASE_URL = args.base_url.rstrip("/")
|
||
|
||
probe = common.actor("probe")
|
||
instances = observe_instances(probe["_token"])
|
||
if len(instances) < 2:
|
||
raise SystemExit("need >= 2 replicas behind LB, observed %s" % instances)
|
||
print("replicas observed through LB: %s" % instances, flush=True)
|
||
rounds = []
|
||
for target in instances:
|
||
print("-- stopping %s (mode=%s) under load" % (target, args.mode), flush=True)
|
||
r = one_round(args, target)
|
||
rounds.append(r)
|
||
w = r["windows"]
|
||
print(" hook %.2fs | LB DOWN after %s s | errors %d (%s) | RPS before %.1f, outage %.1f, after restore %.1f"
|
||
" | p95 outage %s ms | back in rotation %s s | data ok=%s"
|
||
% (r["stop_hook_seconds"], r["lb_detect_down_seconds"], r["errors_during_outage"],
|
||
",".join(r["error_kinds"]) or "-", w["before"]["ok_rps"], w["outage_first_10s"]["ok_rps"],
|
||
w["after_restore"]["ok_rps"], w["outage_first_10s"]["latency_ms_ok"]["p95"],
|
||
r["target_back_in_rotation_seconds"], r["data"]["ok"]), flush=True)
|
||
report = {
|
||
"scenario": "stop each API replica in turn under load, restore before the next",
|
||
"load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0,
|
||
"write_retry": "same Idempotency-Key and body until 201"},
|
||
"replicas": instances,
|
||
"rounds": rounds,
|
||
}
|
||
ok = all(r["data"]["ok"] and r["target_back_in_rotation_seconds"] is not None
|
||
and not r["target_served_after_hook_returned"] for r in rounds)
|
||
report["pass"] = ok
|
||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||
Path(args.output).write_text(text + "\n", encoding="utf-8")
|
||
print(("PASS" if ok else "FAIL") + ": every replica stopped and restored; confirmed data intact, no duplicates")
|
||
sys.exit(0 if ok else 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|