forked from mtusi2026/mtusi2026lab4
106 lines
4.4 KiB
Python
106 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Closed-loop нагрузка POST /messages через LB; во время неё выполняется action-скрипт (остановка реплики).
|
|
Без прозрачных retry: каждая неудача = ошибка. Токены в вывод не попадают."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
from common import call, setup
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--base-url", default="http://localhost:8080")
|
|
ap.add_argument("--duration", type=float, default=40)
|
|
ap.add_argument("--concurrency", type=int, default=8)
|
|
ap.add_argument("--action-script")
|
|
ap.add_argument("--action-at", type=float, default=10)
|
|
ap.add_argument("--target-index", type=int, default=0)
|
|
ap.add_argument("--output")
|
|
a = ap.parse_args()
|
|
|
|
auth, chat = setup(a.base_url)
|
|
path = f"/api/v1/chats/{chat}/messages"
|
|
ids = set()
|
|
for _ in range(40): # какие реплики видны через LB
|
|
_, h, _ = call(a.base_url, "GET", "/api/v1/users/me", headers=auth)
|
|
ids.add(h.get("x-instance-id"))
|
|
ids = sorted(i for i in ids if i)
|
|
|
|
records, lock = [], threading.Lock() # (t, status, latency, instance)
|
|
t_start = time.monotonic()
|
|
t_end = t_start + a.duration
|
|
|
|
def worker():
|
|
while time.monotonic() < t_end:
|
|
t0 = time.monotonic()
|
|
try:
|
|
s, h, _ = call(a.base_url, "POST", path, {"text": "load"},
|
|
{**auth, "Idempotency-Key": uuid.uuid4().hex}, timeout=3)
|
|
inst, ver = h.get("x-instance-id"), h.get("x-app-version")
|
|
except Exception:
|
|
s, inst, ver = 0, None, None
|
|
with lock:
|
|
records.append((t0 - t_start, s, time.monotonic() - t0, inst, ver))
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(a.concurrency)]
|
|
for t in threads:
|
|
t.start()
|
|
|
|
action = {}
|
|
if a.action_script:
|
|
time.sleep(a.action_at)
|
|
target = ids[a.target_index % len(ids)]
|
|
t0 = time.monotonic() - t_start
|
|
rc = subprocess.run([a.action_script], env={**os.environ, "TARGET_INSTANCE_ID": target},
|
|
capture_output=True, timeout=300).returncode
|
|
action = {"target": target, "started_at": round(t0, 2),
|
|
"finished_at": round(time.monotonic() - t_start, 2), "exit_code": rc}
|
|
for t in threads:
|
|
t.join()
|
|
|
|
ok = [r for r in records if r[1] == 201]
|
|
bad = [r for r in records if r[1] != 201]
|
|
lat = sorted(r[2] for r in ok)
|
|
pct = lambda p: round(lat[min(len(lat) - 1, int(len(lat) * p))] * 1000, 1) if lat else None
|
|
summary = {
|
|
"duration_s": a.duration, "concurrency": a.concurrency, "replicas_seen": ids,
|
|
"requests": len(records), "ok": len(ok), "errors": len(bad),
|
|
"error_rate": round(len(bad) / max(1, len(records)), 4),
|
|
"rps_ok": round(len(ok) / a.duration, 1),
|
|
"latency_ms": {"p50": pct(0.50), "p95": pct(0.95), "p99": pct(0.99)},
|
|
"error_statuses": {str(s): sum(1 for r in bad if r[1] == s) for s in {r[1] for r in bad}},
|
|
"action": action,
|
|
}
|
|
if action:
|
|
errs_after = [r[0] for r in bad if r[0] >= action["started_at"]]
|
|
times = sorted(r[0] for r in ok if r[0] >= action["started_at"])
|
|
gaps = [b - a_ for a_, b in zip([action["started_at"]] + times, times)]
|
|
summary["errors_after_action"] = len(errs_after)
|
|
summary["recovery_s"] = round(max(errs_after) - action["started_at"], 2) if errs_after else 0.0
|
|
summary["max_stall_s"] = round(max(gaps), 2) if gaps else None
|
|
summary["instances_in_last_5s"] = sorted({r[3] for r in ok if r[0] >= a.duration - 5 and r[3]})
|
|
|
|
vers = {}
|
|
for r in ok:
|
|
if r[4]:
|
|
v = vers.setdefault(r[4], [r[0], r[0]])
|
|
v[0], v[1] = min(v[0], r[0]), max(v[1], r[0])
|
|
summary["versions_first_last_seen_s"] = {k: [round(x, 1) for x in v] for k, v in vers.items()}
|
|
|
|
timeline = {}
|
|
for t, s, *_ in records:
|
|
b = timeline.setdefault(int(t), {"ok": 0, "err": 0})
|
|
b["ok" if s == 201 else "err"] += 1
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
if a.output:
|
|
with open(a.output, "w") as f:
|
|
json.dump({"summary": summary, "per_second": timeline}, f, ensure_ascii=False, indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |