forked from mtusi2026/mtusi2026lab3
250 lines
12 KiB
Python
250 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Эксперименты уровня 4 (stdlib, Python 3.9+): отказ каждой API-реплики под нагрузкой и rolling update.
|
|
|
|
python3 research/lb_experiments.py failover-series [--mode stop|kill|both]
|
|
python3 research/lb_experiments.py rolling-update
|
|
|
|
Нагрузка: closed-loop, W потоков через LB; каждый поток — свой пользователь, 1 POST сообщения
|
|
(Idempotency-Key; при ошибке/таймауте повтор С ТЕМ ЖЕ ключом и телом) на 3 GET истории.
|
|
Каждая попытка журналируется (время, операция, статус, X-Instance-Id, задержка). Итог — JSON в
|
|
evidence/lb/*.json + текстовая сводка. Токены не сохраняются.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "student-tests"))
|
|
from common import BASE_URL, Client, Failure, login, new_user # noqa: E402
|
|
|
|
OUT = ROOT / "evidence" / "lb"
|
|
DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env"))
|
|
|
|
|
|
def sh(*args, check=True):
|
|
return subprocess.run(list(args), cwd=ROOT, check=check, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def api_containers():
|
|
"""[(container_id, name, instance_id)] запущенных реплик сервиса api."""
|
|
res = []
|
|
for cid in sh(*DC, "ps", "-q", "api").split():
|
|
name = sh("docker", "inspect", "-f", "{{.Name}}", cid).lstrip("/")
|
|
iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False)
|
|
res.append((cid, name, iid))
|
|
return sorted(res, key=lambda x: x[1])
|
|
|
|
|
|
def wait_healthy(cid, timeout=60):
|
|
t = time.monotonic()
|
|
while time.monotonic() - t < timeout:
|
|
if sh("docker", "inspect", "-f", "{{.State.Health.Status}}", cid, check=False) == "healthy":
|
|
return time.monotonic() - t
|
|
time.sleep(0.25)
|
|
raise RuntimeError("container not healthy: " + cid)
|
|
|
|
|
|
class Load:
|
|
def __init__(self, workers, timeout=2.0):
|
|
self.workers, self.timeout = workers, timeout
|
|
self.events, self.lock, self.stop = [], threading.Lock(), threading.Event()
|
|
boot = Client(BASE_URL)
|
|
self.actors = []
|
|
for i in range(workers):
|
|
a = new_user(boot, "lb%d" % i); login(boot, a)
|
|
a["chat"] = boot.call("POST", "/api/v1/chats", 201, actor=a, schema="Chat",
|
|
body={"title": "lb", "member_ids": [a["id"]]})["json"]
|
|
self.actors.append(a)
|
|
self.t0 = None
|
|
self.confirmed = {} # message id -> text (подтверждено 201)
|
|
|
|
def record(self, op, ok, status, instance, latency):
|
|
with self.lock:
|
|
self.events.append((time.monotonic() - self.t0, op, ok, status, instance, latency))
|
|
|
|
def worker(self, a):
|
|
c = Client(BASE_URL, timeout=self.timeout)
|
|
path = "/api/v1/chats/" + a["chat"]["id"] + "/messages"
|
|
n, pending = 0, None
|
|
while not self.stop.is_set():
|
|
n += 1
|
|
if n % 4 == 1:
|
|
if pending is None:
|
|
pending = (uuid.uuid4().hex, "lb-%s-%d" % (a["id"][:8], n))
|
|
key, text = pending
|
|
op, kw = "post", dict(method="POST", path=path, expected=201, actor=a, body={"text": text},
|
|
headers={"Idempotency-Key": key}, schema="Message")
|
|
else:
|
|
op, kw = "get", dict(method="GET", path=path + "?limit=20", expected=200, actor=a, schema="MessagePage")
|
|
t = time.monotonic()
|
|
try:
|
|
r = c.call(**kw)
|
|
self.record(op, True, r["status"], r["headers"].get("X-Instance-Id"), time.monotonic() - t)
|
|
if op == "post":
|
|
with self.lock:
|
|
self.confirmed[r["json"]["id"]] = r["json"]["text"]
|
|
pending = None
|
|
except Failure as exc:
|
|
code = str(exc).rsplit("received ", 1)[-1] if "received" in str(exc) else "conn"
|
|
self.record(op, False, code, None, time.monotonic() - t)
|
|
time.sleep(0.05) # pending остаётся: повтор с тем же ключом и телом
|
|
|
|
def __enter__(self):
|
|
self.t0 = time.monotonic()
|
|
self.threads = [threading.Thread(target=self.worker, args=(a,), daemon=True) for a in self.actors]
|
|
[t.start() for t in self.threads]
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
self.stop.set()
|
|
[t.join(timeout=10) for t in self.threads]
|
|
|
|
def now(self):
|
|
return time.monotonic() - self.t0
|
|
|
|
def verify(self):
|
|
"""Все подтверждённые сообщения на месте, дублей текста (неуверенных повторов) нет."""
|
|
c = Client(BASE_URL, timeout=10)
|
|
texts, ids = [], set()
|
|
for a in self.actors:
|
|
path = "/api/v1/chats/" + a["chat"]["id"] + "/messages?limit=100"
|
|
cursor = None
|
|
while True:
|
|
q = path + ("&cursor=" + quote(cursor, safe="") if cursor else "")
|
|
page = c.call("GET", q, actor=a, schema="MessagePage")["json"]
|
|
for m in page["items"]:
|
|
texts.append(m["text"]); ids.add(m["id"])
|
|
cursor = page["next_cursor"]
|
|
if not cursor:
|
|
break
|
|
lost = [i for i in self.confirmed if i not in ids]
|
|
return {"confirmed": len(self.confirmed), "lost_confirmed": len(lost),
|
|
"stored": len(texts), "duplicate_texts": len(texts) - len(set(texts))}
|
|
|
|
|
|
def window(events, a, b):
|
|
ev = [e for e in events if a <= e[0] < b]
|
|
ok = [e for e in ev if e[2]]
|
|
lat = sorted(e[5] for e in ok)
|
|
pct = (lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1)) if lat else (lambda p: None)
|
|
return {"seconds": round(b - a, 2), "ok": len(ok), "errors": len(ev) - len(ok),
|
|
"rps_ok": round(len(ok) / (b - a), 1) if b > a else None,
|
|
"p50_ms": pct(0.5), "p99_ms": pct(0.99),
|
|
"by_instance": dict(sorted({i: sum(1 for e in ok if e[4] == i) for i in {e[4] for e in ok}}.items())),
|
|
"error_codes": dict(sorted({s: sum(1 for e in ev if not e[2] and e[3] == s)
|
|
for s in {e[3] for e in ev if not e[2]}}.items()))}
|
|
|
|
|
|
def failover_series(args):
|
|
modes = ["stop", "kill"] if args.mode == "both" else [args.mode]
|
|
results = {"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers,
|
|
"client_timeout_s": 2.0, "phases_s": {"before": args.before, "after_action": args.after},
|
|
"runs": []}
|
|
for mode in modes:
|
|
for idx in range(len(api_containers())):
|
|
targets = api_containers()
|
|
cid, name, iid = targets[idx]
|
|
load = Load(args.workers)
|
|
with load:
|
|
time.sleep(args.before)
|
|
t_action = load.now()
|
|
if mode == "stop":
|
|
sh("docker", "stop", "-t", "15", cid) # SIGTERM → drain → exit
|
|
else:
|
|
sh("docker", "kill", "-s", "KILL", cid) # внезапная смерть процесса
|
|
t_done = load.now()
|
|
time.sleep(args.after)
|
|
t_restore = load.now()
|
|
sh("docker", "start", cid)
|
|
healthy_after = wait_healthy(cid)
|
|
new_iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False)
|
|
time.sleep(args.settle)
|
|
t_end = load.now()
|
|
ev = load.events
|
|
errs = [e for e in ev if not e[2] and e[0] >= t_action]
|
|
target_seen = [e[0] for e in ev if e[2] and e[4] == iid and e[0] >= t_action]
|
|
back_seen = [e[0] for e in ev if e[2] and e[4] == new_iid and e[0] >= t_restore]
|
|
posts_ok = sorted(e[0] for e in ev if e[2] and e[1] == "post")
|
|
gaps = [b - a for a, b in zip(posts_ok, posts_ok[1:]) if a >= t_action - 1 and b <= t_restore]
|
|
run = {
|
|
"mode": mode, "container": name, "instance_id": iid,
|
|
"action_seconds": round(t_done - t_action, 2),
|
|
"before": window(ev, 0, t_action), "during": window(ev, t_action, t_restore),
|
|
"after_restore": window(ev, t_restore, t_end),
|
|
"errors_after_action": len(errs),
|
|
"last_error_after_action_s": round(max(e[0] for e in errs) - t_action, 2) if errs else 0,
|
|
"target_last_served_after_action_s": round(max(target_seen) - t_action, 2) if target_seen else 0,
|
|
"max_gap_between_successful_writes_s": round(max(gaps), 2) if gaps else None,
|
|
"replica_healthy_after_start_s": round(healthy_after, 2),
|
|
"restarted_replica_first_served_s": round(min(back_seen) - t_restore, 2) if back_seen else None,
|
|
"data": load.verify(),
|
|
}
|
|
results["runs"].append(run)
|
|
print("%-4s %-18s errors=%-3d last_err=%5.2fs target_last=%5.2fs max_write_gap=%ss rps(before/during)=%s/%s data=%s" % (
|
|
mode, name, run["errors_after_action"], run["last_error_after_action_s"],
|
|
run["target_last_served_after_action_s"], run["max_gap_between_successful_writes_s"],
|
|
run["before"]["rps_ok"], run["during"]["rps_ok"], run["data"]), flush=True)
|
|
time.sleep(2)
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
(OUT / "failover-series.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n")
|
|
print("saved evidence/lb/failover-series.json")
|
|
|
|
|
|
def rolling_update(args):
|
|
before = api_containers()
|
|
images_before = sorted({sh("docker", "inspect", "-f", "{{.Image}}", b[0])[:19] for b in before})
|
|
load = Load(args.workers)
|
|
with load:
|
|
time.sleep(args.before)
|
|
t_a = load.now()
|
|
env = dict(os.environ)
|
|
proc = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
|
|
capture_output=True, text=True)
|
|
t_b = load.now()
|
|
time.sleep(args.after)
|
|
t_end = load.now()
|
|
after = api_containers()
|
|
ev = load.events
|
|
res = {
|
|
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers,
|
|
"script_exit": proc.returncode, "script_seconds": round(t_b - t_a, 2),
|
|
"before_replicas": [{"container": n, "instance_id": i} for _, n, i in before],
|
|
"after_replicas": [{"container": n, "instance_id": i} for _, n, i in after],
|
|
"images_before": images_before,
|
|
"images_after": sorted({sh("docker", "inspect", "-f", "{{.Image}}", a[0])[:19] for a in after}),
|
|
"before": window(ev, 0, t_a), "during": window(ev, t_a, t_b), "after": window(ev, t_b, t_end),
|
|
"data": load.verify(),
|
|
}
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
(OUT / "rolling-update.json").write_text(json.dumps(res, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps({k: res[k] for k in ("script_exit", "script_seconds", "before_replicas", "after_replicas",
|
|
"during", "data")}, ensure_ascii=False, indent=1))
|
|
if proc.returncode:
|
|
print(proc.stderr[-2000:])
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("experiment", choices=["failover-series", "rolling-update"])
|
|
p.add_argument("--mode", choices=["stop", "kill", "both"], default="both")
|
|
p.add_argument("--workers", type=int, default=8)
|
|
p.add_argument("--before", type=float, default=5)
|
|
p.add_argument("--after", type=float, default=10)
|
|
p.add_argument("--settle", type=float, default=5)
|
|
args = p.parse_args()
|
|
{"failover-series": failover_series, "rolling-update": rolling_update}[args.experiment](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|