forked from mtusi2026/mtusi2026lab6
96 lines
4.6 KiB
Python
96 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Уровень 4: rolling update (поочерёдная замена всех реплик) под нагрузкой.
|
|
|
|
Нагрузка (писатели с Idempotency-Key + читатели) идёт всё время; посередине
|
|
выполняется `python scripts/ops.py rolling-update` (сборка образа, однократные
|
|
миграции, +1 новая реплика -> healthy/UP -> graceful stop старой, и так N раз).
|
|
Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые
|
|
реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS,
|
|
ошибки и задержки до/во время/после.
|
|
|
|
python student-tests/rolling_update_load.py --output evidence/rolling_update.json
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import common # noqa: E402
|
|
from failover_series import observe_instances # noqa: E402
|
|
from loadgen import Load # noqa: E402
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main():
|
|
py = sys.executable or "python"
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080"))
|
|
p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404"))
|
|
p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update")
|
|
p.add_argument("--writers", type=int, default=4)
|
|
p.add_argument("--readers", type=int, default=2)
|
|
p.add_argument("--pre", type=float, default=5)
|
|
p.add_argument("--post", type=float, default=8)
|
|
p.add_argument("--output")
|
|
args = p.parse_args()
|
|
cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"]
|
|
common.BASE_URL = args.base_url.rstrip("/")
|
|
|
|
probe = common.actor("rprobe")
|
|
before = observe_instances(probe["_token"])
|
|
print("replicas before: %s" % before, flush=True)
|
|
load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start()
|
|
time.sleep(args.pre)
|
|
t_upd = load.now()
|
|
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200)
|
|
t_done = load.now()
|
|
print(proc.stdout.strip()[-3000:], flush=True)
|
|
time.sleep(args.post)
|
|
load.stop()
|
|
t_end = load.now()
|
|
after = observe_instances(probe["_token"])
|
|
tail_instances = load.window(t_end - 3, t_end)["instances"]
|
|
errs = load.errors_between(t_upd, t_done)
|
|
report = {
|
|
"scenario": "rolling update of all API replicas under continuous load",
|
|
"update_command": " ".join(cmd[1:] if cmd[0] == py else cmd),
|
|
"update_exit_code": proc.returncode,
|
|
"update_seconds": round(t_done - t_upd, 2),
|
|
"replicas_before": before,
|
|
"replicas_after": after,
|
|
"all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)),
|
|
"errors_during_update": len(errs),
|
|
"error_kinds": sorted({str(e[3]) for e in errs}),
|
|
"windows": {
|
|
"before": load.window(0, t_upd),
|
|
"during_update": load.window(t_upd, t_done),
|
|
"after": load.window(t_done, t_end),
|
|
},
|
|
"lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb],
|
|
"data": load.verify(),
|
|
"load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0},
|
|
}
|
|
report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"]
|
|
w = report["windows"]
|
|
print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f"
|
|
" | p95 during %s ms | data ok=%s"
|
|
% (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-",
|
|
w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"],
|
|
w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"]))
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.output).write_text(text + "\n", encoding="utf-8")
|
|
print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load")
|
|
sys.exit(0 if report["pass"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|