#!/usr/bin/env python3 """Нагрузка, поочерёдный отказ каждой реплики и rolling update. Токены не печатает.""" from __future__ import annotations import json import os import platform import subprocess import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure # noqa: E402 def run(cmd, env=None, check=True): return subprocess.run( cmd, cwd=ROOT, env=env, check=check, text=True, capture_output=True, ) def compose(*args): return run(["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args]) def api_ids(): proc = compose("ps", "-q", "api") return [line.strip() for line in proc.stdout.splitlines() if line.strip()] def inspect(cid, fmt): return run(["docker", "inspect", "-f", fmt, cid]).stdout.strip() def percentile(values, p): if not values: return None ordered = sorted(values) index = max(0, int(round((len(ordered) - 1) * p))) return round(ordered[index] * 1000, 3) class Load: def __init__(self, base, actor, chat, concurrency=4): self.base = base self.actor = actor self.chat = chat self.concurrency = concurrency self.events = [] self._stop = threading.Event() self._origin = time.monotonic() self._pool = None self._futures = [] def now(self): return time.monotonic() - self._origin def start(self): def worker(worker_id): local = Client(self.base, timeout=3) index = 0 while not self._stop.is_set(): stamp = self.now() try: response = local.message( self.actor, self.chat, "load-%s-%s-%s" % (worker_id, index, uuid.uuid4().hex[:8]), ) self.events.append({ "ok": True, "t": stamp, "dt": self.now() - stamp, "instance": response["headers"].get("X-Instance-Id"), }) except Failure: self.events.append({ "ok": False, "t": stamp, "dt": self.now() - stamp, "instance": None, }) index += 1 self._pool = ThreadPoolExecutor(max_workers=self.concurrency) self._futures = [self._pool.submit(worker, i) for i in range(self.concurrency)] def stop(self): self._stop.set() for future in self._futures: future.result() self._pool.shutdown(wait=True) def summary(self, start, end): window = [item for item in self.events if start <= item["t"] < end] ok = [item for item in window if item["ok"]] bad = [item for item in window if not item["ok"]] elapsed = max(end - start, 0.001) return { "seconds": round(end - start, 3), "successful_requests": len(ok), "failed_requests": len(bad), "successful_rps": round(len(ok) / elapsed, 3), "error_ratio": round(len(bad) / len(window), 6) if window else None, "success_latency_ms": { "p50": percentile([item["dt"] for item in ok], 0.50), "p95": percentile([item["dt"] for item in ok], 0.95), "p99": percentile([item["dt"] for item in ok], 0.99), }, "instances": sorted({item["instance"] for item in ok if item["instance"]}), } def wait_ready(base, timeout=90): deadline = time.monotonic() + timeout while True: try: Client(base, timeout=2).call("GET", "/health/ready", schema="Health") return except Failure: if time.monotonic() > deadline: raise SystemExit("API did not become ready") time.sleep(0.5) def wait_instance(base, actor, wanted, timeout): started = time.monotonic() seen = set() while time.monotonic() - started < timeout: try: seen.add( Client(base, timeout=2).call( "GET", "/api/v1/users/me", actor=actor, schema="User" )["headers"]["X-Instance-Id"] ) except Failure: pass if wanted in seen: return round(time.monotonic() - started, 3), sorted(seen) time.sleep(0.2) return None, sorted(seen) def container_limits(): rows = [] proc = compose("ps", "-q") for cid in [line.strip() for line in proc.stdout.splitlines() if line.strip()]: rows.append({ "name": inspect(cid, "{{.Name}}").lstrip("/"), "nano_cpus": int(inspect(cid, "{{.HostConfig.NanoCpus}}") or "0"), "memory_bytes": int(inspect(cid, "{{.HostConfig.Memory}}") or "0"), "published": inspect(cid, "{{json .NetworkSettings.Ports}}"), }) return rows def main(): base = os.environ.get("BASE_URL", "http://localhost:8080") wait_ready(base) client = Client(base, timeout=10) alice, _bob, _eve, chat = client.bootstrap() spread = [ client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"] for _ in range(30) ] observed = sorted(set(spread)) if len(observed) < 2: raise SystemExit("need two instance ids, saw %s" % observed) key = "probe-" + uuid.uuid4().hex text = "параллельный ключ " + key barrier = threading.Barrier(8) def once(_n): local = Client(base, timeout=10) barrier.wait() response = local.message(alice, chat, text, key=key) return response["json"]["id"], response["headers"]["X-Instance-Id"] with ThreadPoolExecutor(max_workers=8) as pool: parallel = list(pool.map(once, range(8))) parallel_ids = sorted({item[0] for item in parallel}) parallel_instances = sorted({item[1] for item in parallel}) if len(parallel_ids) != 1: raise SystemExit("parallel idempotency created %s messages" % len(parallel_ids)) load = Load(base, alice, chat) load.start() time.sleep(8) baseline_end = load.now() baseline = load.summary(0, baseline_end) failovers = [] for target in observed: time.sleep(1) mark = load.now() env = os.environ.copy() env["TARGET_INSTANCE_ID"] = target started = time.monotonic() proc = run([str(ROOT / "scripts" / "stop-one.sh")], env=env, check=False) stop_s = round(time.monotonic() - started, 3) stopped_at = load.now() time.sleep(6) window_end = load.now() after = [item for item in load.events if item["t"] >= stopped_at and item["ok"]] served = sorted({item["instance"] for item in after if item["instance"]}) first = None for item in load.events: if item["t"] >= mark and item["ok"] and item["instance"] and item["instance"] != target: first = round(item["t"] - mark, 3) break restore_started = time.monotonic() run([str(ROOT / "scripts" / "restore-api.sh")]) back_s, seen_after = wait_instance(base, alice, target, 40) failovers.append({ "target": target, "stop_hook_exit": proc.returncode, "stop_hook_seconds": stop_s, "window": load.summary(mark, window_end), "first_other_replica_success_seconds": first, "target_seen_after_hook_returned": target in served, "instances_after_hook": served, "restore_until_target_seen_seconds": back_s, "instances_after_restore": seen_after, "restore_command_seconds": round(time.monotonic() - restore_started, 3), }) if proc.returncode != 0: load.stop() raise SystemExit("stop-one failed for %s\n%s" % (target, proc.stderr)) if first is None or first > 10 or target in served: load.stop() raise SystemExit("failover criteria failed for %s" % target) images_before = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()}) roll_mark = load.now() holder = {} def run_roll(): proc = run([str(ROOT / "scripts" / "rolling-update.sh")], check=False) holder["code"] = proc.returncode holder["err"] = (proc.stderr or "")[-800:] holder["out"] = (proc.stdout or "")[-800:] roller = threading.Thread(target=run_roll) roller.start() roller.join() roll_end = load.now() load.stop() images_after = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()}) rolling = load.summary(roll_mark, roll_end) rolling["exit"] = holder.get("code") rolling["stderr_tail"] = holder.get("err") if holder.get("code") != 0: raise SystemExit("rolling update failed: %s\n%s" % (holder.get("err"), holder.get("out"))) report = { "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "platform": platform.platform(), "cpu_count": os.cpu_count(), "workload": "closed-loop POST, concurrency 4, client timeout 3s, no client retry", "observed_instances": observed, "parallel_idempotency": { "requests": 8, "unique_message_ids": len(parallel_ids), "instances": parallel_instances, }, "baseline": baseline, "failovers": failovers, "images_before_roll": images_before, "images_after_roll": images_after, "rolling_under_load": rolling, "container_limits": container_limits(), "lb_timeouts": { "check_interval_seconds": 1, "unhealthy_after_failures": 1, "check_timeout_seconds": 1, "max_detection_seconds": 2, "connect_timeout_seconds": 1, "server_timeout_seconds": 8, "client_timeout_seconds": 8, "drain_seconds": 3, "stop_grace_seconds": 10, }, } dest = ROOT / "evidence" / "grade4.json" dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps({ "observed_instances": observed, "parallel_unique_messages": len(parallel_ids), "parallel_instances": parallel_instances, "baseline_rps": baseline["successful_rps"], "baseline_errors": baseline["failed_requests"], "failovers": [ { "target": item["target"], "stop_s": item["stop_hook_seconds"], "first_other_s": item["first_other_replica_success_seconds"], "errors": item["window"]["failed_requests"], "rps": item["window"]["successful_rps"], "target_seen_after_hook": item["target_seen_after_hook_returned"], "restore_s": item["restore_until_target_seen_seconds"], } for item in failovers ], "rolling_exit": rolling["exit"], "rolling_errors": rolling["failed_requests"], "rolling_rps": rolling["successful_rps"], "images_changed": images_before != images_after, "evidence": "evidence/grade4.json", }, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()