Files
mtusi2026lab5/bench/failover_series.py
T
Alexey BodanyukandClaude Opus 5.5 5112be83cc перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 15:19:00 +03:00

142 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
for detection, only the active health check can find it).
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
Timeline of one round (all times from the moment the stop command is issued):
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
has every replica UP (rejoin time) -> 6 s steady state.
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
request (+s), the first moment after which the stopped replica never answered again
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
traffic again, plus the history check (no confirmed message lost, no duplicate).
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
<output>/series.md (table for REPORT.md). No tokens are written.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import json
import platform
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def docker(*args: str) -> str:
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
load = Load(workers).start()
time.sleep(6)
t_stop = load.now()
if mode == "pause":
docker("pause", replica["id"])
else:
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
t_stopped = load.now()
time.sleep(12)
t_start = load.now()
docker("unpause" if mode == "pause" else "start", replica["id"])
lbctl.wait_up(replicas, timeout=90)
t_up = load.now()
# The restarted container keeps its hostname: wait until it serves requests again.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
break
time.sleep(0.1)
t_serves = load.now()
time.sleep(6)
load.stop()
t_end = load.now()
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
rows = load.per_second()
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
for row in rows:
row = dict(row)
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
writer.writerow(row)
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
return {
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
"stop_command_seconds": round(t_stopped - t_stop, 2),
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
"failed_after_stop": len(failed_after_stop),
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
"target_last_served_after_stop_s": round(out_of_rotation, 3),
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
"client_retries_same_key": load.retries,
"history": load.verify_history(),
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/failover")
parser.add_argument("--modes", default="stop,kill,pause")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
assert len(replicas) >= 2, "need at least two running replicas"
lbctl.wait_up(len(replicas))
results = []
for mode in args.modes.split(","):
for replica in replicas: # each replica in turn; the previous one is already restored
print(f"== {mode} {replica['name']} ...", flush=True)
result = one_round(replica, mode, len(replicas), args.workers, out)
results.append(result)
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
"retry-on conn-failure + redispatch",
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8")
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for r in results:
h = r["history"]
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
*lines, ""]), encoding="utf-8")
print("\n".join(lines))
return 0
if __name__ == "__main__":
raise SystemExit(main())