The k6 profile from the prepared kit runs an open model against the balancer from the edge network: constant arrival rate, 80 % keyed message posts and 20 % history reads over 20 synthetic users and 10 chats, metrics only for the measure phase, and no setup data (tokens) in summary.json. scripts/k6.sh runs it with the host uid so the summary is writable on Linux too. scripts/probe.py gains three things. check --rounds repeats the same-key race with a new key per round and counts rounds with identical bodies, one stored copy and 409 on another body. load runs an action under k6 while a probe posts every 0.1 s and resends the same key after a failure and a poller reads the number of UP servers; it writes the action log, the probe and LB traces, the delta of the backend retry, redispatch and error counters, and a verdict: dip for a replica that leaves and returns, replace for a rolling update that must keep capacity and end on new replicas only. detect runs the action in the background and polls the balancer at the same time, so a drain that ends after the exclusion is visible, and checks the declared maximum, the LB answer after detection and the time to restore. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
437 lines
17 KiB
Python
Executable File
437 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import functools
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
|
|
|
|
|
@functools.cache
|
|
def published(port, fallback):
|
|
command = ["docker", "compose", "port", "lb", str(port)]
|
|
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
|
return found.stdout.strip() or fallback
|
|
|
|
|
|
def base_url():
|
|
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
|
|
|
|
|
def stats_url():
|
|
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
|
|
|
|
|
|
def call(method, path, body=None, headers=None, timeout=2.0):
|
|
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
|
head = {"Accept": "application/json"}
|
|
if data is not None:
|
|
head["Content-Type"] = "application/json"
|
|
head.update(headers or {})
|
|
request = urllib.request.Request(
|
|
base_url().rstrip("/") + path, data=data, headers=head, method=method
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
payload = json.loads(response.read() or b"null")
|
|
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
|
except urllib.error.HTTPError as error:
|
|
try:
|
|
payload = json.loads(error.read() or b"null")
|
|
except (OSError, ValueError):
|
|
payload = None
|
|
return error.code, error.headers.get("X-Instance-Id", ""), payload
|
|
except (OSError, ValueError):
|
|
return 0, "", None
|
|
|
|
|
|
def emit(report, out=None):
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if out:
|
|
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(out).write_text(text + "\n")
|
|
print(text)
|
|
|
|
|
|
def created(result, what):
|
|
if result[0] != 201:
|
|
raise SystemExit(f"{what} failed: HTTP {result[0]}")
|
|
return result[2]
|
|
|
|
|
|
def actor(label):
|
|
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Probe-" + uuid.uuid4().hex
|
|
body = {"username": username, "display_name": "probe " + label, "password": password}
|
|
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
|
|
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
login = call(
|
|
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
|
|
)
|
|
return {
|
|
"id": user["id"],
|
|
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
|
|
}
|
|
|
|
|
|
def chat_of(owner, other):
|
|
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
|
|
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
|
|
|
|
|
|
def send(owner, chat, text, key):
|
|
headers = dict(owner["auth"], **{"Idempotency-Key": key})
|
|
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
|
|
|
|
|
def lb_rows(backend="api"):
|
|
try:
|
|
with urllib.request.urlopen(stats_url(), timeout=2) as response:
|
|
lines = response.read().decode().splitlines()
|
|
except OSError:
|
|
return None
|
|
header = lines[0].lstrip("# ").split(",")
|
|
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
|
|
return [r for r in rows if r["pxname"] == backend]
|
|
|
|
|
|
def lb_up():
|
|
rows = lb_rows()
|
|
if rows is None:
|
|
return None
|
|
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
|
|
|
|
|
|
def lb_counters():
|
|
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
|
|
return {name: int(row.get(name) or 0) for name in COUNTERS}
|
|
|
|
|
|
def run_k6(out, env, on_start=None):
|
|
Path(out).mkdir(parents=True, exist_ok=True)
|
|
started = []
|
|
process = subprocess.Popen(
|
|
[str(ROOT / "scripts/k6.sh"), str(out)],
|
|
env=dict(os.environ, **env),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
)
|
|
with (Path(out) / "k6.log").open("w") as log:
|
|
for line in process.stdout:
|
|
log.write(line)
|
|
if not started and "LOAD-START" in line:
|
|
started.append(time.time())
|
|
if on_start:
|
|
on_start()
|
|
return process.wait(), started[0] if started else None
|
|
|
|
|
|
def k6_metrics(path, duration):
|
|
metrics = json.loads(Path(path).read_text())["metrics"]
|
|
|
|
def value(name, field):
|
|
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
|
|
|
|
count = value("http_reqs", "count")
|
|
return {
|
|
"requests": int(count),
|
|
"rps": round(count / duration, 2),
|
|
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
|
|
"failed_ratio": round(value("http_req_failed", "rate"), 5),
|
|
"dropped_iterations": int(value("dropped_iterations", "count")),
|
|
"p50_ms": round(value("http_req_duration", "med"), 2),
|
|
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
|
|
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
|
|
}
|
|
|
|
|
|
def race(alice, chat, parallel):
|
|
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
|
barrier, results = threading.Barrier(parallel), []
|
|
|
|
def worker():
|
|
barrier.wait()
|
|
results.append(send(alice, chat, text, key))
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(parallel)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
bodies = [r[2] for r in results if r[0] == 201]
|
|
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
|
|
return text, results, identical, send(alice, chat, text + " changed", key)[0]
|
|
|
|
|
|
def check(args):
|
|
alice, bob = actor("alice"), actor("bob")
|
|
chat = chat_of(alice, bob)
|
|
seen = collections.Counter(
|
|
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
|
)
|
|
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
|
|
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
|
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
|
|
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
|
|
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
|
|
same = {
|
|
"rounds": len(rounds),
|
|
"parallel": args.parallel,
|
|
"statuses": dict(statuses),
|
|
"instances": dict(served),
|
|
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
|
|
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
|
|
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
|
|
}
|
|
passed = (
|
|
len(seen) >= 2
|
|
and statuses == {201: len(rounds) * args.parallel}
|
|
and len(served) >= 2
|
|
and same["identical_bodies_rounds"] == len(rounds)
|
|
and same["single_copy_rounds"] == len(rounds)
|
|
and same["conflict_rounds"] == len(rounds)
|
|
)
|
|
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
|
|
return 0 if passed else 1
|
|
|
|
|
|
def record(stop, alice, chat):
|
|
probes, trace = [], []
|
|
|
|
def prober():
|
|
key = uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
started = time.time()
|
|
status, instance, _ = send(alice, chat, "probe " + key, key)
|
|
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
|
if status == 201:
|
|
key = uuid.uuid4().hex
|
|
time.sleep(0.1)
|
|
|
|
def poller():
|
|
while not stop.is_set():
|
|
trace.append((time.time(), lb_up()))
|
|
time.sleep(0.1)
|
|
|
|
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
|
for thread in threads:
|
|
thread.start()
|
|
return probes, trace, threads
|
|
|
|
|
|
def write_csv(path, header, rows):
|
|
with path.open("w") as handle:
|
|
handle.write(header + "\n")
|
|
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
|
|
|
|
|
|
def summarize(args, probes, trace, action, expected):
|
|
t0, end = action["start"], action["end"]
|
|
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
|
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
|
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
|
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
|
outage = [t for t in down if not back or t < back[0]] or [end]
|
|
served = [(t, i) for t, s, _, i in probes if s == 201]
|
|
before = collections.Counter(i for t, i in served if t < t0)
|
|
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
|
|
after = collections.Counter(i for t, i in served if t > end)
|
|
return {
|
|
"label": label,
|
|
"action": args.command,
|
|
"action_exit": action["exit"],
|
|
"action_seconds": round(end - t0, 3),
|
|
"action_finished_before_load_end": end < action["load_end"],
|
|
"lb_up_before": expected,
|
|
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
|
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
|
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
|
"probe": {
|
|
"attempts": len(probes),
|
|
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
|
"first_failure_s": round(failures[0], 3) if failures else None,
|
|
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
|
"instances_before": dict(before),
|
|
"instances_during_outage": dict(during),
|
|
"instances_after": dict(after),
|
|
},
|
|
}
|
|
|
|
|
|
def verdict(args, result):
|
|
probe, k6 = result["probe"], result["k6"]
|
|
checks = {
|
|
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
|
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
|
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
|
}
|
|
if args.expect == "dip":
|
|
checks["detected"] = result["detection_seconds"] is not None
|
|
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
|
|
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
|
|
probe["instances_after"].get(result["label"])
|
|
)
|
|
if args.expect == "replace":
|
|
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
|
|
old, new = set(probe["instances_before"]), set(probe["instances_after"])
|
|
checks["all_replaced"] = bool(new) and not old & new
|
|
return checks
|
|
|
|
|
|
def load(args):
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
expected = lb_up() or 0
|
|
alice, bob = actor("alice"), actor("bob")
|
|
stop, action = threading.Event(), {}
|
|
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
|
|
|
def act():
|
|
time.sleep(args.warmup + args.lead)
|
|
action["start"] = time.time()
|
|
with (out / "action.log").open("w") as log:
|
|
action["exit"] = subprocess.call(
|
|
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
|
|
)
|
|
action["end"] = time.time()
|
|
|
|
runner = threading.Thread(target=act)
|
|
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
|
|
counters = lb_counters()
|
|
code, load_start = run_k6(out / "k6", env, runner.start)
|
|
if load_start is not None:
|
|
runner.join()
|
|
action["load_end"] = load_start + args.warmup + args.duration
|
|
stop.set()
|
|
for thread in threads:
|
|
thread.join()
|
|
if "start" not in action:
|
|
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.log'}")
|
|
t0 = action["start"]
|
|
write_csv(
|
|
out / "probe.csv",
|
|
"t_rel_s,status,latency_ms,instance",
|
|
((round(t - t0, 3), *r) for t, *r in probes),
|
|
)
|
|
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
|
result = summarize(args, probes, trace, action, expected)
|
|
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
|
result["k6"] = dict(
|
|
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
|
)
|
|
result["checks"] = verdict(args, result)
|
|
result["pass"] = all(result["checks"].values())
|
|
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
return 0 if result["pass"] else 1
|
|
|
|
|
|
def detect(args):
|
|
before = lb_up() or 0
|
|
until = before - 1 if args.until is None else args.until
|
|
started = time.time()
|
|
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
|
detected = acted = None
|
|
while time.time() - started < args.timeout and (detected is None or acted is None):
|
|
if acted is None and process.poll() is not None:
|
|
acted = time.time() - started
|
|
if detected is None and (lb_up() or 0) <= until:
|
|
detected = time.time() - started
|
|
time.sleep(0.05)
|
|
code = process.wait()
|
|
acted = time.time() - started if acted is None else acted
|
|
status, instance, payload = call("GET", "/health/live")
|
|
restored = None
|
|
if args.restore:
|
|
begun = time.time()
|
|
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
|
|
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
|
|
time.sleep(0.1)
|
|
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
|
checks = {
|
|
"action_ok": code == 0,
|
|
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
|
"lb_answer_ok": status == args.lb_status,
|
|
}
|
|
if args.within_action:
|
|
checks["excluded_before_action_end"] = detected is not None and detected < acted
|
|
if args.restore:
|
|
checks["restored"] = restored is not None
|
|
report = {
|
|
"action": args.command,
|
|
"action_exit": code,
|
|
"action_seconds": round(acted, 3),
|
|
"lb_up_before": before,
|
|
"lb_up_counted_as_detected": until,
|
|
"detection_seconds": None if detected is None else round(detected, 3),
|
|
"declared_max_seconds": args.max,
|
|
"measurement_slack_seconds": args.slack,
|
|
"lb_answer_after_detection": {
|
|
"path": "/health/live",
|
|
"status": status,
|
|
"x_instance_id_present": bool(instance),
|
|
"error_code": ((payload or {}).get("error") or {}).get("code"),
|
|
},
|
|
"restore": args.restore,
|
|
"restore_seconds": restored,
|
|
"lb_up_after_restore": lb_up(),
|
|
"checks": checks,
|
|
"pass": all(checks.values()),
|
|
}
|
|
emit(report, args.out)
|
|
return 0 if report["pass"] else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
|
sub = parser.add_subparsers(dest="mode", required=True)
|
|
one = sub.add_parser("check", help="instances behind one token and same-key races")
|
|
one.add_argument("--samples", type=int, default=40)
|
|
one.add_argument("--parallel", type=int, default=8)
|
|
one.add_argument("--rounds", type=int, default=1)
|
|
one.add_argument("--out", help="JSON file, written only after a complete report")
|
|
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
|
|
under.add_argument("--out", required=True)
|
|
under.add_argument("--label")
|
|
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
|
|
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
|
|
under.add_argument("--warmup", type=int, default=10)
|
|
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
|
|
under.add_argument("--lead", type=int, default=15)
|
|
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
|
under.add_argument("--max-recovery", type=float, default=10.0)
|
|
under.add_argument("command", nargs=argparse.REMAINDER)
|
|
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
|
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
|
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
|
watch.add_argument("--until", type=int, help="UP servers counted as detected")
|
|
watch.add_argument("--within-action", action="store_true")
|
|
watch.add_argument("--lb-status", type=int, default=200)
|
|
watch.add_argument("--restore", help="shell command that brings replicas back")
|
|
watch.add_argument("--timeout", type=float, default=20)
|
|
watch.add_argument("--out", help="JSON file, written only after a complete report")
|
|
watch.add_argument("command", nargs=argparse.REMAINDER)
|
|
args = parser.parse_args()
|
|
if args.mode == "check":
|
|
return check(args)
|
|
args.command = [c for c in args.command if c != "--"]
|
|
if not args.command:
|
|
parser.error(f"{args.mode} needs an action command after --")
|
|
return load(args) if args.mode == "load" else detect(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|