#!/usr/bin/env python3 """Level 4: stop EACH API replica in turn under load (restoring the previous one first) and measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL) and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case for detection, only the active health check can find it). python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes Timeline of one round (all times from the moment the stop command is issued): warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy has every replica UP (rejoin time) -> 6 s steady state. Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed request (+s), the first moment after which the stopped replica never answered again (= removal from rotation), time until the restarted replica is UP in HAProxy and serves traffic again, plus the history check (no confirmed message lost, no duplicate). Output: /series.json (summary), /timeline--.csv (per second), /series.md (table for REPORT.md). No tokens are written. """ from __future__ import annotations import argparse import csv import datetime as dt import json import platform import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) sys.path.insert(0, str(ROOT / "bench")) import lbctl # noqa: E402 from loadgen import Load # noqa: E402 def docker(*args: str) -> str: return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict: load = Load(workers).start() time.sleep(6) t_stop = load.now() if mode == "pause": docker("pause", replica["id"]) else: docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"]) t_stopped = load.now() time.sleep(12) t_start = load.now() docker("unpause" if mode == "pause" else "start", replica["id"]) lbctl.wait_up(replicas, timeout=90) t_up = load.now() # The restarted container keeps its hostname: wait until it serves requests again. deadline = time.monotonic() + 30 while time.monotonic() < deadline: if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events): break time.sleep(0.1) t_serves = load.now() time.sleep(6) load.stop() t_end = load.now() failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)] served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start] out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0 rows = load.per_second() with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) writer.writeheader() for row in rows: row = dict(row) row["instances"] = row["instances"].replace(replica["hostname"], "TARGET") writer.writerow(row) state = json.loads(docker("inspect", replica["id"]))[0]["State"] return { "mode": mode, "replica": replica["name"], "instance_id": replica["hostname"], "stop_command_seconds": round(t_stopped - t_stop, 2), "before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end), "failed_after_stop": len(failed_after_stop), "failed_statuses": sorted({e[2] for e in failed_after_stop}), "last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3), "target_last_served_after_stop_s": round(out_of_rotation, 3), "rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2), "client_retries_same_key": load.retries, "history": load.verify_history(), "restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}", } def main() -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--output", default="evidence/failover") parser.add_argument("--modes", default="stop,kill,pause") parser.add_argument("--workers", type=int, default=4) args = parser.parse_args() out = ROOT / args.output out.mkdir(parents=True, exist_ok=True) replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"]) assert len(replicas) >= 2, "need at least two running replicas" lbctl.wait_up(len(replicas)) results = [] for mode in args.modes.split(","): for replica in replicas: # each replica in turn; the previous one is already restored print(f"== {mode} {replica['name']} ...", flush=True) result = one_round(replica, mode, len(replicas), args.workers, out) results.append(result) print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s", "target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True) meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip(), "host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}", "workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message " "(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s", "lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; " "retry-on conn-failure + redispatch", "api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"} (out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | " "Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] for r in results: h = r["history"] lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} " f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | " f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | " f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | " f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |") (out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "", *lines, ""]), encoding="utf-8") print("\n".join(lines)) return 0 if __name__ == "__main__": raise SystemExit(main())