#!/usr/bin/env python3 """Уровень 4: rolling update (поочерёдная замена всех реплик) под нагрузкой. Нагрузка (писатели с Idempotency-Key + читатели) идёт всё время; посередине выполняется `python scripts/ops.py rolling-update` (сборка образа, однократные миграции, +1 новая реплика -> healthy/UP -> graceful stop старой, и так N раз). Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS, ошибки и задержки до/во время/после. python student-tests/rolling_update_load.py --output evidence/rolling_update.json """ import argparse import json import os import shlex import subprocess import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import common # noqa: E402 from failover_series import observe_instances # noqa: E402 from loadgen import Load # noqa: E402 ROOT = Path(__file__).resolve().parents[1] def main(): py = sys.executable or "python" p = argparse.ArgumentParser() p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update") p.add_argument("--writers", type=int, default=4) p.add_argument("--readers", type=int, default=2) p.add_argument("--pre", type=float, default=5) p.add_argument("--post", type=float, default=8) p.add_argument("--output") args = p.parse_args() cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"] common.BASE_URL = args.base_url.rstrip("/") probe = common.actor("rprobe") before = observe_instances(probe["_token"]) print("replicas before: %s" % before, flush=True) load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() time.sleep(args.pre) t_upd = load.now() proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200) t_done = load.now() print(proc.stdout.strip()[-3000:], flush=True) time.sleep(args.post) load.stop() t_end = load.now() after = observe_instances(probe["_token"]) tail_instances = load.window(t_end - 3, t_end)["instances"] errs = load.errors_between(t_upd, t_done) report = { "scenario": "rolling update of all API replicas under continuous load", "update_command": " ".join(cmd[1:] if cmd[0] == py else cmd), "update_exit_code": proc.returncode, "update_seconds": round(t_done - t_upd, 2), "replicas_before": before, "replicas_after": after, "all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)), "errors_during_update": len(errs), "error_kinds": sorted({str(e[3]) for e in errs}), "windows": { "before": load.window(0, t_upd), "during_update": load.window(t_upd, t_done), "after": load.window(t_done, t_end), }, "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], "data": load.verify(), "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0}, } report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"] w = report["windows"] print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f" " | p95 during %s ms | data ok=%s" % (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-", w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"], w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"])) text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).parent.mkdir(parents=True, exist_ok=True) Path(args.output).write_text(text + "\n", encoding="utf-8") print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load") sys.exit(0 if report["pass"] else 1) if __name__ == "__main__": main()