Files
mtusi2026lab7-fork/student-tests/rolling_update_load.py

97 lines
4.6 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Уровень 4: rolling update (поочерёдная замена всех реплик) под нагрузкой.
Нагрузка (писатели с Idempotency-Key + читатели) идёт всё время; посередине
выполняется `python scripts/ops.py rolling-update` (сборка образа, однократные
миграции, +1 новая реплика -> healthy/UP -> graceful stop старой, и так N раз).
Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые
реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS,
ошибки и задержки до/во время/после.
python student-tests/rolling_update_load.py --output evidence/rolling_update.json
"""
import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import common # noqa: E402
from failover_series import observe_instances # noqa: E402
from loadgen import Load # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
def main():
py = sys.executable or "python"
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443"))
p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404"))
p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update")
p.add_argument("--writers", type=int, default=4)
p.add_argument("--readers", type=int, default=2)
p.add_argument("--pre", type=float, default=5)
p.add_argument("--post", type=float, default=8)
p.add_argument("--output")
args = p.parse_args()
cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"]
common.BASE_URL = args.base_url.rstrip("/")
probe = common.actor("rprobe")
before = observe_instances(probe["_token"])
print("replicas before: %s" % before, flush=True)
load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start()
time.sleep(args.pre)
t_upd = load.now()
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200)
t_done = load.now()
print(proc.stdout.strip()[-3000:], flush=True)
time.sleep(args.post)
load.stop()
t_end = load.now()
after = observe_instances(probe["_token"])
tail_instances = load.window(t_end - 3, t_end)["instances"]
errs = load.errors_between(t_upd, t_done)
report = {
"scenario": "rolling update of all API replicas under continuous load",
"update_command": " ".join(cmd[1:] if cmd[0] == py else cmd),
"update_exit_code": proc.returncode,
"update_seconds": round(t_done - t_upd, 2),
"replicas_before": before,
"replicas_after": after,
"all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)),
"errors_during_update": len(errs),
"error_kinds": sorted({str(e[3]) for e in errs}),
"windows": {
"before": load.window(0, t_upd),
"during_update": load.window(t_upd, t_done),
"after": load.window(t_done, t_end),
},
"lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb],
"data": load.verify(),
"load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0},
}
report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"]
w = report["windows"]
print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f"
" | p95 during %s ms | data ok=%s"
% (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-",
w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"],
w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"]))
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(text + "\n", encoding="utf-8")
print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load")
sys.exit(0 if report["pass"] else 1)
if __name__ == "__main__":
main()