forked from mtusi2026/mtusi2026lab6
114 lines
6.1 KiB
Python
114 lines
6.1 KiB
Python
#!/usr/bin/env python3
|
||
"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей).
|
||
|
||
Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с):
|
||
t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса)
|
||
t=25 восстановить (scripts/restore-replicas.sh)
|
||
t=40 остановить реплику B
|
||
t=55 восстановить
|
||
Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок
|
||
(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей.
|
||
|
||
python research/failover_sequence.py --out evidence
|
||
"""
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
import loadgen # noqa: E402
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
BASE = "http://127.0.0.1:8080"
|
||
|
||
|
||
def sh(*args):
|
||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||
|
||
|
||
def api_ids():
|
||
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab6",
|
||
"--filter", "label=com.docker.compose.service=api").split()
|
||
|
||
|
||
def scenario(rec, plan):
|
||
"""Выполняется в отдельном потоке: действия по расписанию относительно rec.t0."""
|
||
for at, action in plan:
|
||
time.sleep(max(0, rec.t0 + at - time.time()))
|
||
t = time.time() - rec.t0
|
||
if action == "stop":
|
||
target = api_ids()[0]
|
||
subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True)
|
||
rec.marks.append({"t": round(t, 2), "action": "stop", "container": target})
|
||
else:
|
||
subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True)
|
||
rec.marks.append({"t": round(t, 2), "action": "restore"})
|
||
|
||
|
||
async def main(args):
|
||
stand = loadgen.Stand(BASE)
|
||
await stand.setup()
|
||
rec = loadgen.Recorder()
|
||
rec.marks = []
|
||
plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")]
|
||
threading.Thread(target=scenario, args=(rec, plan), daemon=True).start()
|
||
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
|
||
await asyncio.sleep(2)
|
||
check = await loadgen.verify(stand, rec)
|
||
stops = [m for m in rec.marks if m["action"] == "stop"]
|
||
restores = [m for m in rec.marks if m["action"] == "restore"]
|
||
windows = []
|
||
for i, stop in enumerate(stops):
|
||
t0 = stop["t"]
|
||
errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10]
|
||
last_err = max((e[0] for e in errs), default=t0)
|
||
restore_t = restores[i]["t"] if i < len(restores) else t0 + 15
|
||
windows.append({"stopped_container": stop["container"], "stop_at_s": t0,
|
||
"rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"),
|
||
"rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"),
|
||
"rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"),
|
||
"failed_attempts_within_10s": len(errs),
|
||
"error_statuses": sorted({str(e[2]) for e in errs}),
|
||
"seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0,
|
||
"p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")})
|
||
series = loadgen.per_second(rec)
|
||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows,
|
||
"total": loadgen.summarize(rec), "verification": check,
|
||
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0,
|
||
"timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])}
|
||
for t, b in sorted(series.items())]}
|
||
with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh:
|
||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||
plot(report, os.path.join(args.out, "failover_sequence.png"))
|
||
print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def plot(report, path):
|
||
import matplotlib
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
tl = report["timeline"]
|
||
t = [x["t"] for x in tl]
|
||
fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True)
|
||
ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность")
|
||
ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)")
|
||
ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST")
|
||
for a in ax:
|
||
for m in report["marks"]:
|
||
a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1)
|
||
ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9)
|
||
fig.tight_layout(); fig.savefig(path, dpi=110)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--out", default="evidence")
|
||
ap.add_argument("--seconds", type=int, default=70)
|
||
ap.add_argument("--concurrency", type=int, default=16)
|
||
asyncio.run(main(ap.parse_args()))
|