Files

250 lines
12 KiB
Python

#!/usr/bin/env python3
"""Эксперименты уровня 4 (stdlib, Python 3.9+): отказ каждой API-реплики под нагрузкой и rolling update.
python3 research/lb_experiments.py failover-series [--mode stop|kill|both]
python3 research/lb_experiments.py rolling-update
Нагрузка: closed-loop, W потоков через LB; каждый поток — свой пользователь, 1 POST сообщения
(Idempotency-Key; при ошибке/таймауте повтор С ТЕМ ЖЕ ключом и телом) на 3 GET истории.
Каждая попытка журналируется (время, операция, статус, X-Instance-Id, задержка). Итог — JSON в
evidence/lb/*.json + текстовая сводка. Токены не сохраняются.
"""
import argparse
import json
import os
import shlex
import statistics
import subprocess
import sys
import threading
import time
import uuid
from urllib.parse import quote
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
from common import BASE_URL, Client, Failure, login, new_user # noqa: E402
OUT = ROOT / "evidence" / "lb"
DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env"))
def sh(*args, check=True):
return subprocess.run(list(args), cwd=ROOT, check=check, capture_output=True, text=True).stdout.strip()
def api_containers():
"""[(container_id, name, instance_id)] запущенных реплик сервиса api."""
res = []
for cid in sh(*DC, "ps", "-q", "api").split():
name = sh("docker", "inspect", "-f", "{{.Name}}", cid).lstrip("/")
iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False)
res.append((cid, name, iid))
return sorted(res, key=lambda x: x[1])
def wait_healthy(cid, timeout=60):
t = time.monotonic()
while time.monotonic() - t < timeout:
if sh("docker", "inspect", "-f", "{{.State.Health.Status}}", cid, check=False) == "healthy":
return time.monotonic() - t
time.sleep(0.25)
raise RuntimeError("container not healthy: " + cid)
class Load:
def __init__(self, workers, timeout=2.0):
self.workers, self.timeout = workers, timeout
self.events, self.lock, self.stop = [], threading.Lock(), threading.Event()
boot = Client(BASE_URL)
self.actors = []
for i in range(workers):
a = new_user(boot, "lb%d" % i); login(boot, a)
a["chat"] = boot.call("POST", "/api/v1/chats", 201, actor=a, schema="Chat",
body={"title": "lb", "member_ids": [a["id"]]})["json"]
self.actors.append(a)
self.t0 = None
self.confirmed = {} # message id -> text (подтверждено 201)
def record(self, op, ok, status, instance, latency):
with self.lock:
self.events.append((time.monotonic() - self.t0, op, ok, status, instance, latency))
def worker(self, a):
c = Client(BASE_URL, timeout=self.timeout)
path = "/api/v1/chats/" + a["chat"]["id"] + "/messages"
n, pending = 0, None
while not self.stop.is_set():
n += 1
if n % 4 == 1:
if pending is None:
pending = (uuid.uuid4().hex, "lb-%s-%d" % (a["id"][:8], n))
key, text = pending
op, kw = "post", dict(method="POST", path=path, expected=201, actor=a, body={"text": text},
headers={"Idempotency-Key": key}, schema="Message")
else:
op, kw = "get", dict(method="GET", path=path + "?limit=20", expected=200, actor=a, schema="MessagePage")
t = time.monotonic()
try:
r = c.call(**kw)
self.record(op, True, r["status"], r["headers"].get("X-Instance-Id"), time.monotonic() - t)
if op == "post":
with self.lock:
self.confirmed[r["json"]["id"]] = r["json"]["text"]
pending = None
except Failure as exc:
code = str(exc).rsplit("received ", 1)[-1] if "received" in str(exc) else "conn"
self.record(op, False, code, None, time.monotonic() - t)
time.sleep(0.05) # pending остаётся: повтор с тем же ключом и телом
def __enter__(self):
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self.worker, args=(a,), daemon=True) for a in self.actors]
[t.start() for t in self.threads]
return self
def __exit__(self, *exc):
self.stop.set()
[t.join(timeout=10) for t in self.threads]
def now(self):
return time.monotonic() - self.t0
def verify(self):
"""Все подтверждённые сообщения на месте, дублей текста (неуверенных повторов) нет."""
c = Client(BASE_URL, timeout=10)
texts, ids = [], set()
for a in self.actors:
path = "/api/v1/chats/" + a["chat"]["id"] + "/messages?limit=100"
cursor = None
while True:
q = path + ("&cursor=" + quote(cursor, safe="") if cursor else "")
page = c.call("GET", q, actor=a, schema="MessagePage")["json"]
for m in page["items"]:
texts.append(m["text"]); ids.add(m["id"])
cursor = page["next_cursor"]
if not cursor:
break
lost = [i for i in self.confirmed if i not in ids]
return {"confirmed": len(self.confirmed), "lost_confirmed": len(lost),
"stored": len(texts), "duplicate_texts": len(texts) - len(set(texts))}
def window(events, a, b):
ev = [e for e in events if a <= e[0] < b]
ok = [e for e in ev if e[2]]
lat = sorted(e[5] for e in ok)
pct = (lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1)) if lat else (lambda p: None)
return {"seconds": round(b - a, 2), "ok": len(ok), "errors": len(ev) - len(ok),
"rps_ok": round(len(ok) / (b - a), 1) if b > a else None,
"p50_ms": pct(0.5), "p99_ms": pct(0.99),
"by_instance": dict(sorted({i: sum(1 for e in ok if e[4] == i) for i in {e[4] for e in ok}}.items())),
"error_codes": dict(sorted({s: sum(1 for e in ev if not e[2] and e[3] == s)
for s in {e[3] for e in ev if not e[2]}}.items()))}
def failover_series(args):
modes = ["stop", "kill"] if args.mode == "both" else [args.mode]
results = {"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers,
"client_timeout_s": 2.0, "phases_s": {"before": args.before, "after_action": args.after},
"runs": []}
for mode in modes:
for idx in range(len(api_containers())):
targets = api_containers()
cid, name, iid = targets[idx]
load = Load(args.workers)
with load:
time.sleep(args.before)
t_action = load.now()
if mode == "stop":
sh("docker", "stop", "-t", "15", cid) # SIGTERM → drain → exit
else:
sh("docker", "kill", "-s", "KILL", cid) # внезапная смерть процесса
t_done = load.now()
time.sleep(args.after)
t_restore = load.now()
sh("docker", "start", cid)
healthy_after = wait_healthy(cid)
new_iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False)
time.sleep(args.settle)
t_end = load.now()
ev = load.events
errs = [e for e in ev if not e[2] and e[0] >= t_action]
target_seen = [e[0] for e in ev if e[2] and e[4] == iid and e[0] >= t_action]
back_seen = [e[0] for e in ev if e[2] and e[4] == new_iid and e[0] >= t_restore]
posts_ok = sorted(e[0] for e in ev if e[2] and e[1] == "post")
gaps = [b - a for a, b in zip(posts_ok, posts_ok[1:]) if a >= t_action - 1 and b <= t_restore]
run = {
"mode": mode, "container": name, "instance_id": iid,
"action_seconds": round(t_done - t_action, 2),
"before": window(ev, 0, t_action), "during": window(ev, t_action, t_restore),
"after_restore": window(ev, t_restore, t_end),
"errors_after_action": len(errs),
"last_error_after_action_s": round(max(e[0] for e in errs) - t_action, 2) if errs else 0,
"target_last_served_after_action_s": round(max(target_seen) - t_action, 2) if target_seen else 0,
"max_gap_between_successful_writes_s": round(max(gaps), 2) if gaps else None,
"replica_healthy_after_start_s": round(healthy_after, 2),
"restarted_replica_first_served_s": round(min(back_seen) - t_restore, 2) if back_seen else None,
"data": load.verify(),
}
results["runs"].append(run)
print("%-4s %-18s errors=%-3d last_err=%5.2fs target_last=%5.2fs max_write_gap=%ss rps(before/during)=%s/%s data=%s" % (
mode, name, run["errors_after_action"], run["last_error_after_action_s"],
run["target_last_served_after_action_s"], run["max_gap_between_successful_writes_s"],
run["before"]["rps_ok"], run["during"]["rps_ok"], run["data"]), flush=True)
time.sleep(2)
OUT.mkdir(parents=True, exist_ok=True)
(OUT / "failover-series.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n")
print("saved evidence/lb/failover-series.json")
def rolling_update(args):
before = api_containers()
images_before = sorted({sh("docker", "inspect", "-f", "{{.Image}}", b[0])[:19] for b in before})
load = Load(args.workers)
with load:
time.sleep(args.before)
t_a = load.now()
env = dict(os.environ)
proc = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
capture_output=True, text=True)
t_b = load.now()
time.sleep(args.after)
t_end = load.now()
after = api_containers()
ev = load.events
res = {
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers,
"script_exit": proc.returncode, "script_seconds": round(t_b - t_a, 2),
"before_replicas": [{"container": n, "instance_id": i} for _, n, i in before],
"after_replicas": [{"container": n, "instance_id": i} for _, n, i in after],
"images_before": images_before,
"images_after": sorted({sh("docker", "inspect", "-f", "{{.Image}}", a[0])[:19] for a in after}),
"before": window(ev, 0, t_a), "during": window(ev, t_a, t_b), "after": window(ev, t_b, t_end),
"data": load.verify(),
}
OUT.mkdir(parents=True, exist_ok=True)
(OUT / "rolling-update.json").write_text(json.dumps(res, ensure_ascii=False, indent=2) + "\n")
print(json.dumps({k: res[k] for k in ("script_exit", "script_seconds", "before_replicas", "after_replicas",
"during", "data")}, ensure_ascii=False, indent=1))
if proc.returncode:
print(proc.stderr[-2000:])
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("experiment", choices=["failover-series", "rolling-update"])
p.add_argument("--mode", choices=["stop", "kill", "both"], default="both")
p.add_argument("--workers", type=int, default=8)
p.add_argument("--before", type=float, default=5)
p.add_argument("--after", type=float, default=10)
p.add_argument("--settle", type=float, default=5)
args = p.parse_args()
{"failover-series": failover_series, "rolling-update": rolling_update}[args.experiment](args)
if __name__ == "__main__":
main()