Files
2026-10-30 20:54:04 +03:00

326 lines
11 KiB
Python

#!/usr/bin/env python3
"""Нагрузка, поочерёдный отказ каждой реплики и rolling update. Токены не печатает."""
from __future__ import annotations
import json
import os
import platform
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def run(cmd, env=None, check=True):
return subprocess.run(
cmd,
cwd=ROOT,
env=env,
check=check,
text=True,
capture_output=True,
)
def compose(*args):
return run(["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args])
def api_ids():
proc = compose("ps", "-q", "api")
return [line.strip() for line in proc.stdout.splitlines() if line.strip()]
def inspect(cid, fmt):
return run(["docker", "inspect", "-f", fmt, cid]).stdout.strip()
def percentile(values, p):
if not values:
return None
ordered = sorted(values)
index = max(0, int(round((len(ordered) - 1) * p)))
return round(ordered[index] * 1000, 3)
class Load:
def __init__(self, base, actor, chat, concurrency=4):
self.base = base
self.actor = actor
self.chat = chat
self.concurrency = concurrency
self.events = []
self._stop = threading.Event()
self._origin = time.monotonic()
self._pool = None
self._futures = []
def now(self):
return time.monotonic() - self._origin
def start(self):
def worker(worker_id):
local = Client(self.base, timeout=3)
index = 0
while not self._stop.is_set():
stamp = self.now()
try:
response = local.message(
self.actor,
self.chat,
"load-%s-%s-%s" % (worker_id, index, uuid.uuid4().hex[:8]),
)
self.events.append({
"ok": True,
"t": stamp,
"dt": self.now() - stamp,
"instance": response["headers"].get("X-Instance-Id"),
})
except Failure:
self.events.append({
"ok": False,
"t": stamp,
"dt": self.now() - stamp,
"instance": None,
})
index += 1
self._pool = ThreadPoolExecutor(max_workers=self.concurrency)
self._futures = [self._pool.submit(worker, i) for i in range(self.concurrency)]
def stop(self):
self._stop.set()
for future in self._futures:
future.result()
self._pool.shutdown(wait=True)
def summary(self, start, end):
window = [item for item in self.events if start <= item["t"] < end]
ok = [item for item in window if item["ok"]]
bad = [item for item in window if not item["ok"]]
elapsed = max(end - start, 0.001)
return {
"seconds": round(end - start, 3),
"successful_requests": len(ok),
"failed_requests": len(bad),
"successful_rps": round(len(ok) / elapsed, 3),
"error_ratio": round(len(bad) / len(window), 6) if window else None,
"success_latency_ms": {
"p50": percentile([item["dt"] for item in ok], 0.50),
"p95": percentile([item["dt"] for item in ok], 0.95),
"p99": percentile([item["dt"] for item in ok], 0.99),
},
"instances": sorted({item["instance"] for item in ok if item["instance"]}),
}
def wait_ready(base, timeout=90):
deadline = time.monotonic() + timeout
while True:
try:
Client(base, timeout=2).call("GET", "/health/ready", schema="Health")
return
except Failure:
if time.monotonic() > deadline:
raise SystemExit("API did not become ready")
time.sleep(0.5)
def wait_instance(base, actor, wanted, timeout):
started = time.monotonic()
seen = set()
while time.monotonic() - started < timeout:
try:
seen.add(
Client(base, timeout=2).call(
"GET", "/api/v1/users/me", actor=actor, schema="User"
)["headers"]["X-Instance-Id"]
)
except Failure:
pass
if wanted in seen:
return round(time.monotonic() - started, 3), sorted(seen)
time.sleep(0.2)
return None, sorted(seen)
def container_limits():
rows = []
proc = compose("ps", "-q")
for cid in [line.strip() for line in proc.stdout.splitlines() if line.strip()]:
rows.append({
"name": inspect(cid, "{{.Name}}").lstrip("/"),
"nano_cpus": int(inspect(cid, "{{.HostConfig.NanoCpus}}") or "0"),
"memory_bytes": int(inspect(cid, "{{.HostConfig.Memory}}") or "0"),
"published": inspect(cid, "{{json .NetworkSettings.Ports}}"),
})
return rows
def main():
base = os.environ.get("BASE_URL", "http://localhost:8080")
wait_ready(base)
client = Client(base, timeout=10)
alice, _bob, _eve, chat = client.bootstrap()
spread = [
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
for _ in range(30)
]
observed = sorted(set(spread))
if len(observed) < 2:
raise SystemExit("need two instance ids, saw %s" % observed)
key = "probe-" + uuid.uuid4().hex
text = "параллельный ключ " + key
barrier = threading.Barrier(8)
def once(_n):
local = Client(base, timeout=10)
barrier.wait()
response = local.message(alice, chat, text, key=key)
return response["json"]["id"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=8) as pool:
parallel = list(pool.map(once, range(8)))
parallel_ids = sorted({item[0] for item in parallel})
parallel_instances = sorted({item[1] for item in parallel})
if len(parallel_ids) != 1:
raise SystemExit("parallel idempotency created %s messages" % len(parallel_ids))
load = Load(base, alice, chat)
load.start()
time.sleep(8)
baseline_end = load.now()
baseline = load.summary(0, baseline_end)
failovers = []
for target in observed:
time.sleep(1)
mark = load.now()
env = os.environ.copy()
env["TARGET_INSTANCE_ID"] = target
started = time.monotonic()
proc = run([str(ROOT / "scripts" / "stop-one.sh")], env=env, check=False)
stop_s = round(time.monotonic() - started, 3)
stopped_at = load.now()
time.sleep(6)
window_end = load.now()
after = [item for item in load.events if item["t"] >= stopped_at and item["ok"]]
served = sorted({item["instance"] for item in after if item["instance"]})
first = None
for item in load.events:
if item["t"] >= mark and item["ok"] and item["instance"] and item["instance"] != target:
first = round(item["t"] - mark, 3)
break
restore_started = time.monotonic()
run([str(ROOT / "scripts" / "restore-api.sh")])
back_s, seen_after = wait_instance(base, alice, target, 40)
failovers.append({
"target": target,
"stop_hook_exit": proc.returncode,
"stop_hook_seconds": stop_s,
"window": load.summary(mark, window_end),
"first_other_replica_success_seconds": first,
"target_seen_after_hook_returned": target in served,
"instances_after_hook": served,
"restore_until_target_seen_seconds": back_s,
"instances_after_restore": seen_after,
"restore_command_seconds": round(time.monotonic() - restore_started, 3),
})
if proc.returncode != 0:
load.stop()
raise SystemExit("stop-one failed for %s\n%s" % (target, proc.stderr))
if first is None or first > 10 or target in served:
load.stop()
raise SystemExit("failover criteria failed for %s" % target)
images_before = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()})
roll_mark = load.now()
holder = {}
def run_roll():
proc = run([str(ROOT / "scripts" / "rolling-update.sh")], check=False)
holder["code"] = proc.returncode
holder["err"] = (proc.stderr or "")[-800:]
holder["out"] = (proc.stdout or "")[-800:]
roller = threading.Thread(target=run_roll)
roller.start()
roller.join()
roll_end = load.now()
load.stop()
images_after = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()})
rolling = load.summary(roll_mark, roll_end)
rolling["exit"] = holder.get("code")
rolling["stderr_tail"] = holder.get("err")
if holder.get("code") != 0:
raise SystemExit("rolling update failed: %s\n%s" % (holder.get("err"), holder.get("out")))
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"platform": platform.platform(),
"cpu_count": os.cpu_count(),
"workload": "closed-loop POST, concurrency 4, client timeout 3s, no client retry",
"observed_instances": observed,
"parallel_idempotency": {
"requests": 8,
"unique_message_ids": len(parallel_ids),
"instances": parallel_instances,
},
"baseline": baseline,
"failovers": failovers,
"images_before_roll": images_before,
"images_after_roll": images_after,
"rolling_under_load": rolling,
"container_limits": container_limits(),
"lb_timeouts": {
"check_interval_seconds": 1,
"unhealthy_after_failures": 1,
"check_timeout_seconds": 1,
"max_detection_seconds": 2,
"connect_timeout_seconds": 1,
"server_timeout_seconds": 8,
"client_timeout_seconds": 8,
"drain_seconds": 3,
"stop_grace_seconds": 10,
},
}
dest = ROOT / "evidence" / "grade4.json"
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({
"observed_instances": observed,
"parallel_unique_messages": len(parallel_ids),
"parallel_instances": parallel_instances,
"baseline_rps": baseline["successful_rps"],
"baseline_errors": baseline["failed_requests"],
"failovers": [
{
"target": item["target"],
"stop_s": item["stop_hook_seconds"],
"first_other_s": item["first_other_replica_success_seconds"],
"errors": item["window"]["failed_requests"],
"rps": item["window"]["successful_rps"],
"target_seen_after_hook": item["target_seen_after_hook_returned"],
"restore_s": item["restore_until_target_seen_seconds"],
}
for item in failovers
],
"rolling_exit": rolling["exit"],
"rolling_errors": rolling["failed_requests"],
"rolling_rps": rolling["successful_rps"],
"images_changed": images_before != images_after,
"evidence": "evidence/grade4.json",
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()