#!/usr/bin/env python3 """Эксперименты уровня 4 (stdlib, Python 3.9+): отказ каждой API-реплики под нагрузкой и rolling update. python3 research/lb_experiments.py failover-series [--mode stop|kill|both] python3 research/lb_experiments.py rolling-update Нагрузка: closed-loop, W потоков через LB; каждый поток — свой пользователь, 1 POST сообщения (Idempotency-Key; при ошибке/таймауте повтор С ТЕМ ЖЕ ключом и телом) на 3 GET истории. Каждая попытка журналируется (время, операция, статус, X-Instance-Id, задержка). Итог — JSON в evidence/lb/*.json + текстовая сводка. Токены не сохраняются. """ import argparse import json import os import shlex import statistics import subprocess import sys import threading import time import uuid from urllib.parse import quote from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "student-tests")) from common import BASE_URL, Client, Failure, login, new_user # noqa: E402 OUT = ROOT / "evidence" / "lb" DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env")) def sh(*args, check=True): return subprocess.run(list(args), cwd=ROOT, check=check, capture_output=True, text=True).stdout.strip() def api_containers(): """[(container_id, name, instance_id)] запущенных реплик сервиса api.""" res = [] for cid in sh(*DC, "ps", "-q", "api").split(): name = sh("docker", "inspect", "-f", "{{.Name}}", cid).lstrip("/") iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False) res.append((cid, name, iid)) return sorted(res, key=lambda x: x[1]) def wait_healthy(cid, timeout=60): t = time.monotonic() while time.monotonic() - t < timeout: if sh("docker", "inspect", "-f", "{{.State.Health.Status}}", cid, check=False) == "healthy": return time.monotonic() - t time.sleep(0.25) raise RuntimeError("container not healthy: " + cid) class Load: def __init__(self, workers, timeout=2.0): self.workers, self.timeout = workers, timeout self.events, self.lock, self.stop = [], threading.Lock(), threading.Event() boot = Client(BASE_URL) self.actors = [] for i in range(workers): a = new_user(boot, "lb%d" % i); login(boot, a) a["chat"] = boot.call("POST", "/api/v1/chats", 201, actor=a, schema="Chat", body={"title": "lb", "member_ids": [a["id"]]})["json"] self.actors.append(a) self.t0 = None self.confirmed = {} # message id -> text (подтверждено 201) def record(self, op, ok, status, instance, latency): with self.lock: self.events.append((time.monotonic() - self.t0, op, ok, status, instance, latency)) def worker(self, a): c = Client(BASE_URL, timeout=self.timeout) path = "/api/v1/chats/" + a["chat"]["id"] + "/messages" n, pending = 0, None while not self.stop.is_set(): n += 1 if n % 4 == 1: if pending is None: pending = (uuid.uuid4().hex, "lb-%s-%d" % (a["id"][:8], n)) key, text = pending op, kw = "post", dict(method="POST", path=path, expected=201, actor=a, body={"text": text}, headers={"Idempotency-Key": key}, schema="Message") else: op, kw = "get", dict(method="GET", path=path + "?limit=20", expected=200, actor=a, schema="MessagePage") t = time.monotonic() try: r = c.call(**kw) self.record(op, True, r["status"], r["headers"].get("X-Instance-Id"), time.monotonic() - t) if op == "post": with self.lock: self.confirmed[r["json"]["id"]] = r["json"]["text"] pending = None except Failure as exc: code = str(exc).rsplit("received ", 1)[-1] if "received" in str(exc) else "conn" self.record(op, False, code, None, time.monotonic() - t) time.sleep(0.05) # pending остаётся: повтор с тем же ключом и телом def __enter__(self): self.t0 = time.monotonic() self.threads = [threading.Thread(target=self.worker, args=(a,), daemon=True) for a in self.actors] [t.start() for t in self.threads] return self def __exit__(self, *exc): self.stop.set() [t.join(timeout=10) for t in self.threads] def now(self): return time.monotonic() - self.t0 def verify(self): """Все подтверждённые сообщения на месте, дублей текста (неуверенных повторов) нет.""" c = Client(BASE_URL, timeout=10) texts, ids = [], set() for a in self.actors: path = "/api/v1/chats/" + a["chat"]["id"] + "/messages?limit=100" cursor = None while True: q = path + ("&cursor=" + quote(cursor, safe="") if cursor else "") page = c.call("GET", q, actor=a, schema="MessagePage")["json"] for m in page["items"]: texts.append(m["text"]); ids.add(m["id"]) cursor = page["next_cursor"] if not cursor: break lost = [i for i in self.confirmed if i not in ids] return {"confirmed": len(self.confirmed), "lost_confirmed": len(lost), "stored": len(texts), "duplicate_texts": len(texts) - len(set(texts))} def window(events, a, b): ev = [e for e in events if a <= e[0] < b] ok = [e for e in ev if e[2]] lat = sorted(e[5] for e in ok) pct = (lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1)) if lat else (lambda p: None) return {"seconds": round(b - a, 2), "ok": len(ok), "errors": len(ev) - len(ok), "rps_ok": round(len(ok) / (b - a), 1) if b > a else None, "p50_ms": pct(0.5), "p99_ms": pct(0.99), "by_instance": dict(sorted({i: sum(1 for e in ok if e[4] == i) for i in {e[4] for e in ok}}.items())), "error_codes": dict(sorted({s: sum(1 for e in ev if not e[2] and e[3] == s) for s in {e[3] for e in ev if not e[2]}}.items()))} def failover_series(args): modes = ["stop", "kill"] if args.mode == "both" else [args.mode] results = {"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers, "client_timeout_s": 2.0, "phases_s": {"before": args.before, "after_action": args.after}, "runs": []} for mode in modes: for idx in range(len(api_containers())): targets = api_containers() cid, name, iid = targets[idx] load = Load(args.workers) with load: time.sleep(args.before) t_action = load.now() if mode == "stop": sh("docker", "stop", "-t", "15", cid) # SIGTERM → drain → exit else: sh("docker", "kill", "-s", "KILL", cid) # внезапная смерть процесса t_done = load.now() time.sleep(args.after) t_restore = load.now() sh("docker", "start", cid) healthy_after = wait_healthy(cid) new_iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False) time.sleep(args.settle) t_end = load.now() ev = load.events errs = [e for e in ev if not e[2] and e[0] >= t_action] target_seen = [e[0] for e in ev if e[2] and e[4] == iid and e[0] >= t_action] back_seen = [e[0] for e in ev if e[2] and e[4] == new_iid and e[0] >= t_restore] posts_ok = sorted(e[0] for e in ev if e[2] and e[1] == "post") gaps = [b - a for a, b in zip(posts_ok, posts_ok[1:]) if a >= t_action - 1 and b <= t_restore] run = { "mode": mode, "container": name, "instance_id": iid, "action_seconds": round(t_done - t_action, 2), "before": window(ev, 0, t_action), "during": window(ev, t_action, t_restore), "after_restore": window(ev, t_restore, t_end), "errors_after_action": len(errs), "last_error_after_action_s": round(max(e[0] for e in errs) - t_action, 2) if errs else 0, "target_last_served_after_action_s": round(max(target_seen) - t_action, 2) if target_seen else 0, "max_gap_between_successful_writes_s": round(max(gaps), 2) if gaps else None, "replica_healthy_after_start_s": round(healthy_after, 2), "restarted_replica_first_served_s": round(min(back_seen) - t_restore, 2) if back_seen else None, "data": load.verify(), } results["runs"].append(run) print("%-4s %-18s errors=%-3d last_err=%5.2fs target_last=%5.2fs max_write_gap=%ss rps(before/during)=%s/%s data=%s" % ( mode, name, run["errors_after_action"], run["last_error_after_action_s"], run["target_last_served_after_action_s"], run["max_gap_between_successful_writes_s"], run["before"]["rps_ok"], run["during"]["rps_ok"], run["data"]), flush=True) time.sleep(2) OUT.mkdir(parents=True, exist_ok=True) (OUT / "failover-series.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n") print("saved evidence/lb/failover-series.json") def rolling_update(args): before = api_containers() images_before = sorted({sh("docker", "inspect", "-f", "{{.Image}}", b[0])[:19] for b in before}) load = Load(args.workers) with load: time.sleep(args.before) t_a = load.now() env = dict(os.environ) proc = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env, capture_output=True, text=True) t_b = load.now() time.sleep(args.after) t_end = load.now() after = api_containers() ev = load.events res = { "measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers, "script_exit": proc.returncode, "script_seconds": round(t_b - t_a, 2), "before_replicas": [{"container": n, "instance_id": i} for _, n, i in before], "after_replicas": [{"container": n, "instance_id": i} for _, n, i in after], "images_before": images_before, "images_after": sorted({sh("docker", "inspect", "-f", "{{.Image}}", a[0])[:19] for a in after}), "before": window(ev, 0, t_a), "during": window(ev, t_a, t_b), "after": window(ev, t_b, t_end), "data": load.verify(), } OUT.mkdir(parents=True, exist_ok=True) (OUT / "rolling-update.json").write_text(json.dumps(res, ensure_ascii=False, indent=2) + "\n") print(json.dumps({k: res[k] for k in ("script_exit", "script_seconds", "before_replicas", "after_replicas", "during", "data")}, ensure_ascii=False, indent=1)) if proc.returncode: print(proc.stderr[-2000:]) def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("experiment", choices=["failover-series", "rolling-update"]) p.add_argument("--mode", choices=["stop", "kill", "both"], default="both") p.add_argument("--workers", type=int, default=8) p.add_argument("--before", type=float, default=5) p.add_argument("--after", type=float, default=10) p.add_argument("--settle", type=float, default=5) args = p.parse_args() {"failover-series": failover_series, "rolling-update": rolling_update}[args.experiment](args) if __name__ == "__main__": main()