Files
mtusi2026lab5-fork/student-tests/rolling_update_load.py

96 lines
4.6 KiB
Python

#!/usr/bin/env python3
"""Уровень 4: rolling update (поочерёдная замена всех реплик) под нагрузкой.
Нагрузка (писатели с Idempotency-Key + читатели) идёт всё время; посередине
выполняется `python scripts/ops.py rolling-update` (сборка образа, однократные
миграции, +1 новая реплика -> healthy/UP -> graceful stop старой, и так N раз).
Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые
реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS,
ошибки и задержки до/во время/после.
python student-tests/rolling_update_load.py --output evidence/rolling_update.json
"""
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import common # noqa: E402
from failover_series import observe_instances # noqa: E402
from loadgen import Load # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
def main():
py = sys.executable or "python"
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080"))
p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404"))
p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update")
p.add_argument("--writers", type=int, default=4)
p.add_argument("--readers", type=int, default=2)
p.add_argument("--pre", type=float, default=5)
p.add_argument("--post", type=float, default=8)
p.add_argument("--output")
args = p.parse_args()
cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"]
common.BASE_URL = args.base_url.rstrip("/")
probe = common.actor("rprobe")
before = observe_instances(probe["_token"])
print("replicas before: %s" % before, flush=True)
load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start()
time.sleep(args.pre)
t_upd = load.now()
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200)
t_done = load.now()
print(proc.stdout.strip()[-3000:], flush=True)
time.sleep(args.post)
load.stop()
t_end = load.now()
after = observe_instances(probe["_token"])
tail_instances = load.window(t_end - 3, t_end)["instances"]
errs = load.errors_between(t_upd, t_done)
report = {
"scenario": "rolling update of all API replicas under continuous load",
"update_command": " ".join(cmd[1:] if cmd[0] == py else cmd),
"update_exit_code": proc.returncode,
"update_seconds": round(t_done - t_upd, 2),
"replicas_before": before,
"replicas_after": after,
"all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)),
"errors_during_update": len(errs),
"error_kinds": sorted({str(e[3]) for e in errs}),
"windows": {
"before": load.window(0, t_upd),
"during_update": load.window(t_upd, t_done),
"after": load.window(t_done, t_end),
},
"lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb],
"data": load.verify(),
"load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0},
}
report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"]
w = report["windows"]
print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f"
" | p95 during %s ms | data ok=%s"
% (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-",
w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"],
w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"]))
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(text + "\n", encoding="utf-8")
print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load")
sys.exit(0 if report["pass"] else 1)
if __name__ == "__main__":
main()