forked from mtusi2026/mtusi2026lab7
97 lines
4.6 KiB
Python
97 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
||
"""Уровень 4: rolling update (поочерёдная замена всех реплик) под нагрузкой.
|
||
|
||
Нагрузка (писатели с Idempotency-Key + читатели) идёт всё время; посередине
|
||
выполняется `python scripts/ops.py rolling-update` (сборка образа, однократные
|
||
миграции, +1 новая реплика -> healthy/UP -> graceful stop старой, и так N раз).
|
||
Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые
|
||
реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS,
|
||
ошибки и задержки до/во время/после.
|
||
|
||
python student-tests/rolling_update_load.py --output evidence/rolling_update.json
|
||
"""
|
||
import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда
|
||
import argparse
|
||
import json
|
||
import os
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
import common # noqa: E402
|
||
from failover_series import observe_instances # noqa: E402
|
||
from loadgen import Load # noqa: E402
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
|
||
|
||
def main():
|
||
py = sys.executable or "python"
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443"))
|
||
p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404"))
|
||
p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update")
|
||
p.add_argument("--writers", type=int, default=4)
|
||
p.add_argument("--readers", type=int, default=2)
|
||
p.add_argument("--pre", type=float, default=5)
|
||
p.add_argument("--post", type=float, default=8)
|
||
p.add_argument("--output")
|
||
args = p.parse_args()
|
||
cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"]
|
||
common.BASE_URL = args.base_url.rstrip("/")
|
||
|
||
probe = common.actor("rprobe")
|
||
before = observe_instances(probe["_token"])
|
||
print("replicas before: %s" % before, flush=True)
|
||
load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start()
|
||
time.sleep(args.pre)
|
||
t_upd = load.now()
|
||
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200)
|
||
t_done = load.now()
|
||
print(proc.stdout.strip()[-3000:], flush=True)
|
||
time.sleep(args.post)
|
||
load.stop()
|
||
t_end = load.now()
|
||
after = observe_instances(probe["_token"])
|
||
tail_instances = load.window(t_end - 3, t_end)["instances"]
|
||
errs = load.errors_between(t_upd, t_done)
|
||
report = {
|
||
"scenario": "rolling update of all API replicas under continuous load",
|
||
"update_command": " ".join(cmd[1:] if cmd[0] == py else cmd),
|
||
"update_exit_code": proc.returncode,
|
||
"update_seconds": round(t_done - t_upd, 2),
|
||
"replicas_before": before,
|
||
"replicas_after": after,
|
||
"all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)),
|
||
"errors_during_update": len(errs),
|
||
"error_kinds": sorted({str(e[3]) for e in errs}),
|
||
"windows": {
|
||
"before": load.window(0, t_upd),
|
||
"during_update": load.window(t_upd, t_done),
|
||
"after": load.window(t_done, t_end),
|
||
},
|
||
"lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb],
|
||
"data": load.verify(),
|
||
"load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0},
|
||
}
|
||
report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"]
|
||
w = report["windows"]
|
||
print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f"
|
||
" | p95 during %s ms | data ok=%s"
|
||
% (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-",
|
||
w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"],
|
||
w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"]))
|
||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||
Path(args.output).write_text(text + "\n", encoding="utf-8")
|
||
print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load")
|
||
sys.exit(0 if report["pass"] else 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|