forked from mtusi2026/mtusi2026lab4
Исходники, Dockerfile, compose.yaml, скрипты и собственные тесты скопированы из mtusi2026lab3 (ветка solution/bvt2302-bodanyuk, b6847e2) как есть; изменения lab4 — отдельными коммитами. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
142 lines
7.6 KiB
Python
142 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
|
|
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
|
|
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
|
|
for detection, only the active health check can find it).
|
|
|
|
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
|
|
|
|
Timeline of one round (all times from the moment the stop command is issued):
|
|
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
|
|
has every replica UP (rejoin time) -> 6 s steady state.
|
|
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
|
|
request (+s), the first moment after which the stopped replica never answered again
|
|
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
|
|
traffic again, plus the history check (no confirmed message lost, no duplicate).
|
|
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
|
|
<output>/series.md (table for REPORT.md). No tokens are written.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import datetime as dt
|
|
import json
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
sys.path.insert(0, str(ROOT / "bench"))
|
|
import lbctl # noqa: E402
|
|
from loadgen import Load # noqa: E402
|
|
|
|
|
|
def docker(*args: str) -> str:
|
|
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
|
|
|
|
|
|
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
|
|
load = Load(workers).start()
|
|
time.sleep(6)
|
|
t_stop = load.now()
|
|
if mode == "pause":
|
|
docker("pause", replica["id"])
|
|
else:
|
|
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
|
|
t_stopped = load.now()
|
|
time.sleep(12)
|
|
t_start = load.now()
|
|
docker("unpause" if mode == "pause" else "start", replica["id"])
|
|
lbctl.wait_up(replicas, timeout=90)
|
|
t_up = load.now()
|
|
# The restarted container keeps its hostname: wait until it serves requests again.
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
|
|
break
|
|
time.sleep(0.1)
|
|
t_serves = load.now()
|
|
time.sleep(6)
|
|
load.stop()
|
|
t_end = load.now()
|
|
|
|
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
|
|
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
|
|
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
|
|
rows = load.per_second()
|
|
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
|
|
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
|
|
writer.writeheader()
|
|
for row in rows:
|
|
row = dict(row)
|
|
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
|
|
writer.writerow(row)
|
|
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
|
|
return {
|
|
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
|
|
"stop_command_seconds": round(t_stopped - t_stop, 2),
|
|
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
|
|
"failed_after_stop": len(failed_after_stop),
|
|
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
|
|
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
|
|
"target_last_served_after_stop_s": round(out_of_rotation, 3),
|
|
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
|
|
"client_retries_same_key": load.retries,
|
|
"history": load.verify_history(),
|
|
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
parser.add_argument("--output", default="evidence/failover")
|
|
parser.add_argument("--modes", default="stop,kill,pause")
|
|
parser.add_argument("--workers", type=int, default=4)
|
|
args = parser.parse_args()
|
|
out = ROOT / args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
|
|
assert len(replicas) >= 2, "need at least two running replicas"
|
|
lbctl.wait_up(len(replicas))
|
|
results = []
|
|
for mode in args.modes.split(","):
|
|
for replica in replicas: # each replica in turn; the previous one is already restored
|
|
print(f"== {mode} {replica['name']} ...", flush=True)
|
|
result = one_round(replica, mode, len(replicas), args.workers, out)
|
|
results.append(result)
|
|
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
|
|
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
|
|
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
|
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
|
text=True).stdout.strip(),
|
|
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
|
|
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
|
|
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
|
|
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
|
|
"retry-on conn-failure + redispatch",
|
|
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
|
|
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8")
|
|
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
|
|
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
|
|
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
|
|
for r in results:
|
|
h = r["history"]
|
|
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
|
|
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
|
|
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
|
|
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
|
|
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
|
|
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
|
|
*lines, ""]), encoding="utf-8")
|
|
print("\n".join(lines))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|