Carry sourcef591fa64e5forward from72a1e57. Preserve lab4 attachment transactions, reconciliation locks, quarantine, and existing schema data through verified legacy adoption. Keep report and evidence work in progress untouched.
593 lines
24 KiB
Python
Executable File
593 lines
24 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import functools
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
|
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
|
|
PSQL += ["-X", "-q", "-At"]
|
|
|
|
|
|
@functools.cache
|
|
def published(port, fallback):
|
|
command = ["docker", "compose", "port", "lb", str(port)]
|
|
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
|
return found.stdout.strip() or fallback
|
|
|
|
|
|
def base_url():
|
|
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
|
|
|
|
|
def stats_url():
|
|
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
|
|
|
|
|
|
def call(method, path, body=None, headers=None, timeout=2.0):
|
|
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
|
head = {"Accept": "application/json"}
|
|
if data is not None:
|
|
head["Content-Type"] = "application/json"
|
|
head.update(headers or {})
|
|
request = urllib.request.Request(
|
|
base_url().rstrip("/") + path, data=data, headers=head, method=method
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
payload = json.loads(response.read() or b"null")
|
|
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
|
except urllib.error.HTTPError as error:
|
|
try:
|
|
payload = json.loads(error.read() or b"null")
|
|
except (OSError, ValueError):
|
|
payload = None
|
|
return error.code, error.headers.get("X-Instance-Id", ""), payload
|
|
except (OSError, ValueError):
|
|
return 0, "", None
|
|
|
|
|
|
def emit(report, out=None):
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if out:
|
|
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(out).write_text(text + "\n")
|
|
print(text)
|
|
|
|
|
|
def created(result, what):
|
|
if result[0] != 201:
|
|
raise SystemExit(f"{what} failed: HTTP {result[0]}")
|
|
return result[2]
|
|
|
|
|
|
def actor(label):
|
|
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Probe-" + uuid.uuid4().hex
|
|
body = {"username": username, "display_name": "probe " + label, "password": password}
|
|
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
|
|
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
login = call(
|
|
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
|
|
)
|
|
return {
|
|
"id": user["id"],
|
|
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
|
|
}
|
|
|
|
|
|
def chat_of(owner, other):
|
|
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
|
|
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
|
|
|
|
|
|
def send(owner, chat, text, key):
|
|
headers = dict(owner["auth"], **{"Idempotency-Key": key})
|
|
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
|
|
|
|
|
def lb_rows(backend="api"):
|
|
try:
|
|
with urllib.request.urlopen(stats_url(), timeout=2) as response:
|
|
lines = response.read().decode().splitlines()
|
|
except OSError:
|
|
return None
|
|
header = lines[0].lstrip("# ").split(",")
|
|
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
|
|
return [r for r in rows if r["pxname"] == backend]
|
|
|
|
|
|
def lb_up():
|
|
rows = lb_rows()
|
|
if rows is None:
|
|
return None
|
|
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
|
|
|
|
|
|
def lb_counters():
|
|
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
|
|
return {name: int(row.get(name) or 0) for name in COUNTERS}
|
|
|
|
|
|
def run_k6(out, env, on_start=None):
|
|
Path(out).mkdir(parents=True, exist_ok=True)
|
|
started = []
|
|
process = subprocess.Popen(
|
|
[str(ROOT / "scripts/k6.sh"), str(out)],
|
|
env=dict(os.environ, **env),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
)
|
|
with (Path(out) / "k6.txt").open("w") as log:
|
|
for line in process.stdout:
|
|
log.write(line)
|
|
if not started and "LOAD-START" in line:
|
|
started.append(time.time())
|
|
if on_start:
|
|
on_start()
|
|
return process.wait(), started[0] if started else None
|
|
|
|
|
|
def k6_metrics(path, duration):
|
|
metrics = json.loads(Path(path).read_text())["metrics"]
|
|
|
|
def value(name, field):
|
|
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
|
|
|
|
count = value("http_reqs", "count")
|
|
return {
|
|
"requests": int(count),
|
|
"rps": round(count / duration, 2),
|
|
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
|
|
"failed_ratio": round(value("http_req_failed", "rate"), 5),
|
|
"dropped_iterations": int(value("dropped_iterations", "count")),
|
|
"p50_ms": round(value("http_req_duration", "med"), 2),
|
|
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
|
|
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
|
|
}
|
|
|
|
|
|
def race(alice, chat, parallel):
|
|
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
|
barrier, results = threading.Barrier(parallel), []
|
|
|
|
def worker():
|
|
barrier.wait()
|
|
results.append(send(alice, chat, text, key))
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(parallel)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
bodies = [r[2] for r in results if r[0] == 201]
|
|
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
|
|
return text, results, identical, send(alice, chat, text + " changed", key)[0]
|
|
|
|
|
|
def check(args):
|
|
alice, bob = actor("alice"), actor("bob")
|
|
chat = chat_of(alice, bob)
|
|
seen = collections.Counter(
|
|
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
|
)
|
|
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
|
|
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
|
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
|
|
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
|
|
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
|
|
same = {
|
|
"rounds": len(rounds),
|
|
"parallel": args.parallel,
|
|
"statuses": dict(statuses),
|
|
"instances": dict(served),
|
|
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
|
|
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
|
|
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
|
|
}
|
|
passed = (
|
|
len(seen) >= 2
|
|
and statuses == {201: len(rounds) * args.parallel}
|
|
and len(served) >= 2
|
|
and same["identical_bodies_rounds"] == len(rounds)
|
|
and same["single_copy_rounds"] == len(rounds)
|
|
and same["conflict_rounds"] == len(rounds)
|
|
)
|
|
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
|
|
return 0 if passed else 1
|
|
|
|
|
|
def output(*command):
|
|
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
|
return done.stdout + done.stderr
|
|
|
|
|
|
def replicas():
|
|
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
|
|
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
|
|
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
|
|
|
|
|
|
def answered(container, request_id):
|
|
for line in output("docker", "logs", container).splitlines():
|
|
if request_id in line and "request.handled" in line:
|
|
return json.loads(line).get("status")
|
|
return None
|
|
|
|
|
|
def lost(alice, bob, chat, kill, args):
|
|
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
|
|
request_id = "lost-" + key[:12]
|
|
lock = subprocess.Popen(
|
|
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
|
)
|
|
lock.stdin.write(
|
|
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
|
|
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
|
|
)
|
|
lock.stdin.flush()
|
|
lock.stdout.readline()
|
|
sent = time.time()
|
|
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
|
|
path = f"/api/v1/chats/{chat}/messages"
|
|
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
|
|
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
|
|
waiting = set(output(*PSQL, "-c", sql).split())
|
|
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
|
|
if kill and holder[0]:
|
|
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
|
|
lock.stdin.close()
|
|
lock.wait()
|
|
retried = time.time()
|
|
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
|
|
while status != 201 and attempts < 20:
|
|
time.sleep(0.2)
|
|
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
|
|
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
|
|
copies = [m for m in page["items"] if m["text"] == text]
|
|
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
|
|
report = {
|
|
"variant": "kill" if kill else "timeout",
|
|
"lost_attempt": {
|
|
"status": first,
|
|
"client_timeout_s": args.timeout,
|
|
"chat_lock_held_s": args.hold,
|
|
"replica": holder[1],
|
|
"replica_killed": bool(kill and holder[0]),
|
|
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
|
|
},
|
|
"retry": {
|
|
"after_s": round(retried - sent, 3),
|
|
"status": status,
|
|
"attempts": attempts,
|
|
"replica": instance,
|
|
},
|
|
"copies_in_history": len(copies),
|
|
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
|
|
"written_by": None
|
|
if stamp is None
|
|
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
|
|
}
|
|
if kill and holder[0]:
|
|
subprocess.call(
|
|
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
|
)
|
|
report["pass"] = (
|
|
first == 0
|
|
and holder[0] is not None
|
|
and status == 201
|
|
and report["copies_in_history"] == 1
|
|
and report["same_message"]
|
|
and (kill or report["lost_attempt"]["replica_answered"] == 201)
|
|
)
|
|
return report
|
|
|
|
|
|
def retry(args):
|
|
alice, bob = actor("alice"), actor("bob")
|
|
chat = chat_of(alice, bob)
|
|
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
|
|
passed = all(v["pass"] for v in variants)
|
|
emit({"variants": variants, "pass": passed}, args.out)
|
|
return 0 if passed else 1
|
|
|
|
|
|
def record(stop, alice, chat):
|
|
probes, trace, confirmed = [], [], []
|
|
|
|
def prober():
|
|
key = uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
started = time.time()
|
|
status, instance, body = send(alice, chat, "probe " + key, key)
|
|
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
|
if status == 201:
|
|
confirmed.append(body)
|
|
key = uuid.uuid4().hex
|
|
time.sleep(0.1)
|
|
|
|
def poller():
|
|
while not stop.is_set():
|
|
trace.append((time.time(), lb_up()))
|
|
time.sleep(0.1)
|
|
|
|
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
|
for thread in threads:
|
|
thread.start()
|
|
return probes, trace, threads, confirmed
|
|
|
|
|
|
def history_check(reader, chat, confirmed):
|
|
items, cursors = [], set()
|
|
cursor = None
|
|
for _ in range(len(confirmed) // 100 + 2):
|
|
path = f"/api/v1/chats/{chat}/messages?limit=100"
|
|
if cursor:
|
|
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
|
|
status, _, page = call("GET", path, headers=reader["auth"])
|
|
if status != 200 or not isinstance(page, dict):
|
|
break
|
|
items.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
|
|
return {
|
|
"confirmed": len(confirmed),
|
|
"stored": len(items),
|
|
"exact_match": bool(confirmed) and same,
|
|
"unique": len({m["id"] for m in items}) == len(items),
|
|
}
|
|
if cursor in cursors:
|
|
break
|
|
cursors.add(cursor)
|
|
return {
|
|
"confirmed": len(confirmed),
|
|
"stored": len(items),
|
|
"exact_match": False,
|
|
"unique": False,
|
|
}
|
|
|
|
|
|
def write_csv(path, header, rows):
|
|
with path.open("w") as handle:
|
|
handle.write(header + "\n")
|
|
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
|
|
|
|
|
|
def summarize(args, probes, trace, action, expected):
|
|
t0, end = action["start"], action["end"]
|
|
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
|
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
|
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
|
|
recovery = (
|
|
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
|
|
)
|
|
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
|
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
|
outage = [t for t in down if not back or t < back[0]] or [end]
|
|
served = [(t, i) for t, s, _, i in probes if s == 201]
|
|
before = collections.Counter(i for t, i in served if t < t0)
|
|
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
|
|
after = collections.Counter(i for t, i in served if t > end)
|
|
return {
|
|
"label": label,
|
|
"action": args.command,
|
|
"action_exit": action["exit"],
|
|
"action_seconds": round(end - t0, 3),
|
|
"action_finished_before_load_end": end < action["load_end"],
|
|
"lb_up_before": expected,
|
|
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
|
"lb_samples_missing": sum(up is None for _, up in trace),
|
|
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
|
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
|
"probe": {
|
|
"attempts": len(probes),
|
|
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
|
"first_failure_s": round(failures[0], 3) if failures else None,
|
|
"first_success_s": round(successes[0][1], 3) if successes else None,
|
|
"recovery_s": round(recovery, 3) if recovery is not None else None,
|
|
"instances_before": dict(before),
|
|
"instances_during_outage": dict(during),
|
|
"instances_after": dict(after),
|
|
},
|
|
}
|
|
|
|
|
|
def verdict(args, result):
|
|
probe, k6 = result["probe"], result["k6"]
|
|
checks = {
|
|
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
|
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
|
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
|
|
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
|
|
"write_recovery_ok": all(
|
|
probe[k] is not None and probe[k] <= args.max_recovery
|
|
for k in ("first_success_s", "recovery_s")
|
|
),
|
|
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
|
|
}
|
|
if args.expect == "dip":
|
|
checks["detected"] = result["detection_seconds"] is not None
|
|
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
|
|
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
|
|
probe["instances_after"].get(result["label"])
|
|
)
|
|
if args.expect == "replace":
|
|
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
|
|
old, new = set(probe["instances_before"]), set(probe["instances_after"])
|
|
checks["all_replaced"] = bool(new) and not old & new
|
|
return checks
|
|
|
|
|
|
def load(args):
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
expected = lb_up() or 0
|
|
alice, bob = actor("alice"), actor("bob")
|
|
stop, action = threading.Event(), {}
|
|
chat = chat_of(alice, bob)
|
|
probes, trace, threads, confirmed = record(stop, alice, chat)
|
|
|
|
def act():
|
|
time.sleep(args.warmup + args.lead)
|
|
action["start"] = time.time()
|
|
with (out / "action.txt").open("w") as log:
|
|
action["exit"] = subprocess.call(
|
|
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
|
|
)
|
|
action["end"] = time.time()
|
|
|
|
runner = threading.Thread(target=act)
|
|
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
|
|
counters = lb_counters()
|
|
code, load_start = run_k6(out / "k6", env, runner.start)
|
|
if load_start is not None:
|
|
runner.join()
|
|
action["load_end"] = load_start + args.warmup + args.duration
|
|
stop.set()
|
|
for thread in threads:
|
|
thread.join()
|
|
if "start" not in action:
|
|
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
|
|
t0 = action["start"]
|
|
write_csv(
|
|
out / "probe.csv",
|
|
"t_rel_s,status,latency_ms,instance",
|
|
((round(t - t0, 3), *r) for t, *r in probes),
|
|
)
|
|
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
|
result = summarize(args, probes, trace, action, expected)
|
|
result["history"] = history_check(bob, chat, confirmed)
|
|
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
|
result["k6"] = dict(
|
|
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
|
)
|
|
result["checks"] = verdict(args, result)
|
|
result["pass"] = all(result["checks"].values())
|
|
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
return 0 if result["pass"] else 1
|
|
|
|
|
|
def detect(args):
|
|
before = lb_up() or 0
|
|
until = before - 1 if args.until is None else args.until
|
|
started = time.time()
|
|
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
|
detected = acted = None
|
|
missing = 0
|
|
while time.time() - started < args.timeout and (detected is None or acted is None):
|
|
if acted is None and process.poll() is not None:
|
|
acted = time.time() - started
|
|
up = lb_up()
|
|
missing += up is None
|
|
if detected is None and up is not None and up <= until:
|
|
detected = time.time() - started
|
|
time.sleep(0.05)
|
|
code = process.wait()
|
|
acted = time.time() - started if acted is None else acted
|
|
status, instance, payload = call("GET", "/health/live")
|
|
restored = None
|
|
if args.restore:
|
|
begun = time.time()
|
|
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
|
|
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
|
|
time.sleep(0.1)
|
|
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
|
checks = {
|
|
"action_ok": code == 0,
|
|
"lb_observed": before >= 2 and missing == 0,
|
|
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
|
"lb_answer_ok": status == args.lb_status,
|
|
}
|
|
if args.within_action:
|
|
checks["excluded_before_action_end"] = detected is not None and detected < acted
|
|
if args.restore:
|
|
checks["restored"] = restored is not None
|
|
report = {
|
|
"action": args.command,
|
|
"action_exit": code,
|
|
"action_seconds": round(acted, 3),
|
|
"lb_up_before": before,
|
|
"lb_samples_missing": missing,
|
|
"lb_up_counted_as_detected": until,
|
|
"detection_seconds": None if detected is None else round(detected, 3),
|
|
"declared_max_seconds": args.max,
|
|
"measurement_slack_seconds": args.slack,
|
|
"lb_answer_after_detection": {
|
|
"path": "/health/live",
|
|
"status": status,
|
|
"x_instance_id_present": bool(instance),
|
|
"error_code": ((payload or {}).get("error") or {}).get("code"),
|
|
},
|
|
"restore": args.restore,
|
|
"restore_seconds": restored,
|
|
"lb_up_after_restore": lb_up(),
|
|
"checks": checks,
|
|
"pass": all(checks.values()),
|
|
}
|
|
emit(report, args.out)
|
|
return 0 if report["pass"] else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
|
sub = parser.add_subparsers(dest="mode", required=True)
|
|
one = sub.add_parser("check", help="instances behind one token and same-key races")
|
|
one.add_argument("--samples", type=int, default=40)
|
|
one.add_argument("--parallel", type=int, default=8)
|
|
one.add_argument("--rounds", type=int, default=1)
|
|
one.add_argument("--out", help="JSON file, written only after a complete report")
|
|
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
|
|
under.add_argument("--out", required=True)
|
|
under.add_argument("--label")
|
|
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
|
|
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
|
|
under.add_argument("--warmup", type=int, default=10)
|
|
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
|
|
under.add_argument("--lead", type=int, default=15)
|
|
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
|
under.add_argument("--max-recovery", type=float, default=10.0)
|
|
under.add_argument("command", nargs=argparse.REMAINDER)
|
|
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
|
|
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
|
|
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
|
|
again.add_argument("--out", help="JSON file, written only after a complete report")
|
|
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
|
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
|
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
|
watch.add_argument("--until", type=int, help="UP servers counted as detected")
|
|
watch.add_argument("--within-action", action="store_true")
|
|
watch.add_argument("--lb-status", type=int, default=200)
|
|
watch.add_argument("--restore", help="shell command that brings replicas back")
|
|
watch.add_argument("--timeout", type=float, default=20)
|
|
watch.add_argument("--out", help="JSON file, written only after a complete report")
|
|
watch.add_argument("command", nargs=argparse.REMAINDER)
|
|
args = parser.parse_args()
|
|
if args.mode == "check":
|
|
return check(args)
|
|
if args.mode == "retry":
|
|
return retry(args)
|
|
args.command = [c for c in args.command if c != "--"]
|
|
if not args.command:
|
|
parser.error(f"{args.mode} needs an action command after --")
|
|
return load(args) if args.mode == "load" else detect(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|