Files
mtusi2026lab5/research/failover_sequence.py
T

114 lines
6.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей).
Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с):
t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса)
t=25 восстановить (scripts/restore-replicas.sh)
t=40 остановить реплику B
t=55 восстановить
Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок
(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей.
python research/failover_sequence.py --out evidence
"""
import argparse
import asyncio
import json
import os
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
def sh(*args):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
def api_ids():
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab5",
"--filter", "label=com.docker.compose.service=api").split()
def scenario(rec, plan):
"""Выполняется в отдельном потоке: действия по расписанию относительно rec.t0."""
for at, action in plan:
time.sleep(max(0, rec.t0 + at - time.time()))
t = time.time() - rec.t0
if action == "stop":
target = api_ids()[0]
subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True)
rec.marks.append({"t": round(t, 2), "action": "stop", "container": target})
else:
subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True)
rec.marks.append({"t": round(t, 2), "action": "restore"})
async def main(args):
stand = loadgen.Stand(BASE)
await stand.setup()
rec = loadgen.Recorder()
rec.marks = []
plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")]
threading.Thread(target=scenario, args=(rec, plan), daemon=True).start()
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
await asyncio.sleep(2)
check = await loadgen.verify(stand, rec)
stops = [m for m in rec.marks if m["action"] == "stop"]
restores = [m for m in rec.marks if m["action"] == "restore"]
windows = []
for i, stop in enumerate(stops):
t0 = stop["t"]
errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10]
last_err = max((e[0] for e in errs), default=t0)
restore_t = restores[i]["t"] if i < len(restores) else t0 + 15
windows.append({"stopped_container": stop["container"], "stop_at_s": t0,
"rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"),
"rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"),
"rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"),
"failed_attempts_within_10s": len(errs),
"error_statuses": sorted({str(e[2]) for e in errs}),
"seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0,
"p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")})
series = loadgen.per_second(rec)
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows,
"total": loadgen.summarize(rec), "verification": check,
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0,
"timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])}
for t, b in sorted(series.items())]}
with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "failover_sequence.png"))
print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
tl = report["timeline"]
t = [x["t"] for x in tl]
fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True)
ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность")
ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)")
ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST")
for a in ax:
for m in report["marks"]:
a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1)
ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9)
fig.tight_layout(); fig.savefig(path, dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--seconds", type=int, default=70)
ap.add_argument("--concurrency", type=int, default=16)
asyncio.run(main(ap.parse_args()))