No failover run ever lost an answer, so the retry of an unconfirmed write was only covered in process. make retry-check holds the chat row with SELECT ... FOR UPDATE for 1.5 s (below lock_timeout), sends a message with a 0.5 s client timeout, finds the replica whose request waits on the lock in pg_stat_activity and then retries with the same key and text through the LB, once with that replica alive and once after docker kill. It checks the 201, a single copy with the id of the retry answer and, for the live replica, the 201 it logged without a client; created_at tells which attempt wrote the message. CI runs it. failover-each and rolling-update now record the database size and a docker stats snapshot in the background of the action, rolling-update names the tag and id of every image it replaces, and scaling.py writes started and the compose-up log marks in UTC with Z; the track README states that the recorded rows are host time (MSK) and what host-others.csv covers. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
541 lines
22 KiB
Python
Executable File
541 lines
22 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import functools
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
|
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
|
|
PSQL += ["-X", "-q", "-At"]
|
|
|
|
|
|
@functools.cache
|
|
def published(port, fallback):
|
|
command = ["docker", "compose", "port", "lb", str(port)]
|
|
found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
|
return found.stdout.strip() or fallback
|
|
|
|
|
|
def base_url():
|
|
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
|
|
|
|
|
def stats_url():
|
|
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
|
|
|
|
|
|
def call(method, path, body=None, headers=None, timeout=2.0):
|
|
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
|
head = {"Accept": "application/json"}
|
|
if data is not None:
|
|
head["Content-Type"] = "application/json"
|
|
head.update(headers or {})
|
|
request = urllib.request.Request(
|
|
base_url().rstrip("/") + path, data=data, headers=head, method=method
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
payload = json.loads(response.read() or b"null")
|
|
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
|
except urllib.error.HTTPError as error:
|
|
try:
|
|
payload = json.loads(error.read() or b"null")
|
|
except (OSError, ValueError):
|
|
payload = None
|
|
return error.code, error.headers.get("X-Instance-Id", ""), payload
|
|
except (OSError, ValueError):
|
|
return 0, "", None
|
|
|
|
|
|
def emit(report, out=None):
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if out:
|
|
Path(out).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(out).write_text(text + "\n")
|
|
print(text)
|
|
|
|
|
|
def created(result, what):
|
|
if result[0] != 201:
|
|
raise SystemExit(f"{what} failed: HTTP {result[0]}")
|
|
return result[2]
|
|
|
|
|
|
def actor(label):
|
|
username = f"p_{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Probe-" + uuid.uuid4().hex
|
|
body = {"username": username, "display_name": "probe " + label, "password": password}
|
|
user = created(call("POST", "/api/v1/users", body, timeout=10), "register")
|
|
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
login = call(
|
|
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10
|
|
)
|
|
return {
|
|
"id": user["id"],
|
|
"auth": {"Authorization": "Bearer " + created(login, "login")["token"]},
|
|
}
|
|
|
|
|
|
def chat_of(owner, other):
|
|
body = {"title": "probe", "member_ids": [owner["id"], other["id"]]}
|
|
return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"]
|
|
|
|
|
|
def send(owner, chat, text, key):
|
|
headers = dict(owner["auth"], **{"Idempotency-Key": key})
|
|
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
|
|
|
|
|
def lb_rows(backend="api"):
|
|
try:
|
|
with urllib.request.urlopen(stats_url(), timeout=2) as response:
|
|
lines = response.read().decode().splitlines()
|
|
except OSError:
|
|
return None
|
|
header = lines[0].lstrip("# ").split(",")
|
|
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
|
|
return [r for r in rows if r["pxname"] == backend]
|
|
|
|
|
|
def lb_up():
|
|
rows = lb_rows()
|
|
if rows is None:
|
|
return None
|
|
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
|
|
|
|
|
|
def lb_counters():
|
|
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
|
|
return {name: int(row.get(name) or 0) for name in COUNTERS}
|
|
|
|
|
|
def run_k6(out, env, on_start=None):
|
|
Path(out).mkdir(parents=True, exist_ok=True)
|
|
started = []
|
|
process = subprocess.Popen(
|
|
[str(ROOT / "scripts/k6.sh"), str(out)],
|
|
env=dict(os.environ, **env),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
)
|
|
with (Path(out) / "k6.txt").open("w") as log:
|
|
for line in process.stdout:
|
|
log.write(line)
|
|
if not started and "LOAD-START" in line:
|
|
started.append(time.time())
|
|
if on_start:
|
|
on_start()
|
|
return process.wait(), started[0] if started else None
|
|
|
|
|
|
def k6_metrics(path, duration):
|
|
metrics = json.loads(Path(path).read_text())["metrics"]
|
|
|
|
def value(name, field):
|
|
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
|
|
|
|
count = value("http_reqs", "count")
|
|
return {
|
|
"requests": int(count),
|
|
"rps": round(count / duration, 2),
|
|
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
|
|
"failed_ratio": round(value("http_req_failed", "rate"), 5),
|
|
"dropped_iterations": int(value("dropped_iterations", "count")),
|
|
"p50_ms": round(value("http_req_duration", "med"), 2),
|
|
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
|
|
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
|
|
}
|
|
|
|
|
|
def race(alice, chat, parallel):
|
|
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
|
barrier, results = threading.Barrier(parallel), []
|
|
|
|
def worker():
|
|
barrier.wait()
|
|
results.append(send(alice, chat, text, key))
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(parallel)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
bodies = [r[2] for r in results if r[0] == 201]
|
|
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
|
|
return text, results, identical, send(alice, chat, text + " changed", key)[0]
|
|
|
|
|
|
def check(args):
|
|
alice, bob = actor("alice"), actor("bob")
|
|
chat = chat_of(alice, bob)
|
|
seen = collections.Counter(
|
|
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
|
)
|
|
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
|
|
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
|
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
|
|
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
|
|
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
|
|
same = {
|
|
"rounds": len(rounds),
|
|
"parallel": args.parallel,
|
|
"statuses": dict(statuses),
|
|
"instances": dict(served),
|
|
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
|
|
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
|
|
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
|
|
}
|
|
passed = (
|
|
len(seen) >= 2
|
|
and statuses == {201: len(rounds) * args.parallel}
|
|
and len(served) >= 2
|
|
and same["identical_bodies_rounds"] == len(rounds)
|
|
and same["single_copy_rounds"] == len(rounds)
|
|
and same["conflict_rounds"] == len(rounds)
|
|
)
|
|
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
|
|
return 0 if passed else 1
|
|
|
|
|
|
def output(*command):
|
|
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
|
return done.stdout + done.stderr
|
|
|
|
|
|
def replicas():
|
|
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
|
|
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
|
|
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
|
|
|
|
|
|
def answered(container, request_id):
|
|
for line in output("docker", "logs", container).splitlines():
|
|
if request_id in line and "request.handled" in line:
|
|
return json.loads(line).get("status")
|
|
return None
|
|
|
|
|
|
def lost(alice, bob, chat, kill, args):
|
|
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
|
|
request_id = "lost-" + key[:12]
|
|
lock = subprocess.Popen(
|
|
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
|
)
|
|
lock.stdin.write(
|
|
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
|
|
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
|
|
)
|
|
lock.stdin.flush()
|
|
lock.stdout.readline()
|
|
sent = time.time()
|
|
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
|
|
path = f"/api/v1/chats/{chat}/messages"
|
|
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
|
|
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
|
|
waiting = set(output(*PSQL, "-c", sql).split())
|
|
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
|
|
if kill and holder[0]:
|
|
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
|
|
lock.stdin.close()
|
|
lock.wait()
|
|
retried = time.time()
|
|
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
|
|
while status != 201 and attempts < 20:
|
|
time.sleep(0.2)
|
|
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
|
|
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
|
|
copies = [m for m in page["items"] if m["text"] == text]
|
|
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
|
|
report = {
|
|
"variant": "kill" if kill else "timeout",
|
|
"lost_attempt": {
|
|
"status": first,
|
|
"client_timeout_s": args.timeout,
|
|
"chat_lock_held_s": args.hold,
|
|
"replica": holder[1],
|
|
"replica_killed": bool(kill and holder[0]),
|
|
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
|
|
},
|
|
"retry": {
|
|
"after_s": round(retried - sent, 3),
|
|
"status": status,
|
|
"attempts": attempts,
|
|
"replica": instance,
|
|
},
|
|
"copies_in_history": len(copies),
|
|
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
|
|
"written_by": None
|
|
if stamp is None
|
|
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
|
|
}
|
|
if kill and holder[0]:
|
|
subprocess.call(
|
|
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
|
)
|
|
report["pass"] = (
|
|
first == 0
|
|
and holder[0] is not None
|
|
and status == 201
|
|
and report["copies_in_history"] == 1
|
|
and report["same_message"]
|
|
and (kill or report["lost_attempt"]["replica_answered"] == 201)
|
|
)
|
|
return report
|
|
|
|
|
|
def retry(args):
|
|
alice, bob = actor("alice"), actor("bob")
|
|
chat = chat_of(alice, bob)
|
|
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
|
|
passed = all(v["pass"] for v in variants)
|
|
emit({"variants": variants, "pass": passed}, args.out)
|
|
return 0 if passed else 1
|
|
|
|
|
|
def record(stop, alice, chat):
|
|
probes, trace = [], []
|
|
|
|
def prober():
|
|
key = uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
started = time.time()
|
|
status, instance, _ = send(alice, chat, "probe " + key, key)
|
|
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
|
if status == 201:
|
|
key = uuid.uuid4().hex
|
|
time.sleep(0.1)
|
|
|
|
def poller():
|
|
while not stop.is_set():
|
|
trace.append((time.time(), lb_up()))
|
|
time.sleep(0.1)
|
|
|
|
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
|
for thread in threads:
|
|
thread.start()
|
|
return probes, trace, threads
|
|
|
|
|
|
def write_csv(path, header, rows):
|
|
with path.open("w") as handle:
|
|
handle.write(header + "\n")
|
|
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
|
|
|
|
|
|
def summarize(args, probes, trace, action, expected):
|
|
t0, end = action["start"], action["end"]
|
|
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
|
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
|
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
|
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
|
outage = [t for t in down if not back or t < back[0]] or [end]
|
|
served = [(t, i) for t, s, _, i in probes if s == 201]
|
|
before = collections.Counter(i for t, i in served if t < t0)
|
|
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
|
|
after = collections.Counter(i for t, i in served if t > end)
|
|
return {
|
|
"label": label,
|
|
"action": args.command,
|
|
"action_exit": action["exit"],
|
|
"action_seconds": round(end - t0, 3),
|
|
"action_finished_before_load_end": end < action["load_end"],
|
|
"lb_up_before": expected,
|
|
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
|
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
|
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
|
"probe": {
|
|
"attempts": len(probes),
|
|
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
|
"first_failure_s": round(failures[0], 3) if failures else None,
|
|
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
|
"instances_before": dict(before),
|
|
"instances_during_outage": dict(during),
|
|
"instances_after": dict(after),
|
|
},
|
|
}
|
|
|
|
|
|
def verdict(args, result):
|
|
probe, k6 = result["probe"], result["k6"]
|
|
checks = {
|
|
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
|
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
|
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
|
}
|
|
if args.expect == "dip":
|
|
checks["detected"] = result["detection_seconds"] is not None
|
|
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
|
|
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
|
|
probe["instances_after"].get(result["label"])
|
|
)
|
|
if args.expect == "replace":
|
|
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
|
|
old, new = set(probe["instances_before"]), set(probe["instances_after"])
|
|
checks["all_replaced"] = bool(new) and not old & new
|
|
return checks
|
|
|
|
|
|
def load(args):
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
expected = lb_up() or 0
|
|
alice, bob = actor("alice"), actor("bob")
|
|
stop, action = threading.Event(), {}
|
|
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
|
|
|
def act():
|
|
time.sleep(args.warmup + args.lead)
|
|
action["start"] = time.time()
|
|
with (out / "action.txt").open("w") as log:
|
|
action["exit"] = subprocess.call(
|
|
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
|
|
)
|
|
action["end"] = time.time()
|
|
|
|
runner = threading.Thread(target=act)
|
|
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
|
|
counters = lb_counters()
|
|
code, load_start = run_k6(out / "k6", env, runner.start)
|
|
if load_start is not None:
|
|
runner.join()
|
|
action["load_end"] = load_start + args.warmup + args.duration
|
|
stop.set()
|
|
for thread in threads:
|
|
thread.join()
|
|
if "start" not in action:
|
|
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.txt'}")
|
|
t0 = action["start"]
|
|
write_csv(
|
|
out / "probe.csv",
|
|
"t_rel_s,status,latency_ms,instance",
|
|
((round(t - t0, 3), *r) for t, *r in probes),
|
|
)
|
|
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
|
result = summarize(args, probes, trace, action, expected)
|
|
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
|
result["k6"] = dict(
|
|
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
|
)
|
|
result["checks"] = verdict(args, result)
|
|
result["pass"] = all(result["checks"].values())
|
|
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
return 0 if result["pass"] else 1
|
|
|
|
|
|
def detect(args):
|
|
before = lb_up() or 0
|
|
until = before - 1 if args.until is None else args.until
|
|
started = time.time()
|
|
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
|
detected = acted = None
|
|
while time.time() - started < args.timeout and (detected is None or acted is None):
|
|
if acted is None and process.poll() is not None:
|
|
acted = time.time() - started
|
|
if detected is None and (lb_up() or 0) <= until:
|
|
detected = time.time() - started
|
|
time.sleep(0.05)
|
|
code = process.wait()
|
|
acted = time.time() - started if acted is None else acted
|
|
status, instance, payload = call("GET", "/health/live")
|
|
restored = None
|
|
if args.restore:
|
|
begun = time.time()
|
|
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
|
|
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
|
|
time.sleep(0.1)
|
|
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
|
checks = {
|
|
"action_ok": code == 0,
|
|
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
|
"lb_answer_ok": status == args.lb_status,
|
|
}
|
|
if args.within_action:
|
|
checks["excluded_before_action_end"] = detected is not None and detected < acted
|
|
if args.restore:
|
|
checks["restored"] = restored is not None
|
|
report = {
|
|
"action": args.command,
|
|
"action_exit": code,
|
|
"action_seconds": round(acted, 3),
|
|
"lb_up_before": before,
|
|
"lb_up_counted_as_detected": until,
|
|
"detection_seconds": None if detected is None else round(detected, 3),
|
|
"declared_max_seconds": args.max,
|
|
"measurement_slack_seconds": args.slack,
|
|
"lb_answer_after_detection": {
|
|
"path": "/health/live",
|
|
"status": status,
|
|
"x_instance_id_present": bool(instance),
|
|
"error_code": ((payload or {}).get("error") or {}).get("code"),
|
|
},
|
|
"restore": args.restore,
|
|
"restore_seconds": restored,
|
|
"lb_up_after_restore": lb_up(),
|
|
"checks": checks,
|
|
"pass": all(checks.values()),
|
|
}
|
|
emit(report, args.out)
|
|
return 0 if report["pass"] else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
|
sub = parser.add_subparsers(dest="mode", required=True)
|
|
one = sub.add_parser("check", help="instances behind one token and same-key races")
|
|
one.add_argument("--samples", type=int, default=40)
|
|
one.add_argument("--parallel", type=int, default=8)
|
|
one.add_argument("--rounds", type=int, default=1)
|
|
one.add_argument("--out", help="JSON file, written only after a complete report")
|
|
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
|
|
under.add_argument("--out", required=True)
|
|
under.add_argument("--label")
|
|
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
|
|
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
|
|
under.add_argument("--warmup", type=int, default=10)
|
|
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
|
|
under.add_argument("--lead", type=int, default=15)
|
|
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
|
under.add_argument("--max-recovery", type=float, default=10.0)
|
|
under.add_argument("command", nargs=argparse.REMAINDER)
|
|
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
|
|
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
|
|
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
|
|
again.add_argument("--out", help="JSON file, written only after a complete report")
|
|
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
|
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
|
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
|
watch.add_argument("--until", type=int, help="UP servers counted as detected")
|
|
watch.add_argument("--within-action", action="store_true")
|
|
watch.add_argument("--lb-status", type=int, default=200)
|
|
watch.add_argument("--restore", help="shell command that brings replicas back")
|
|
watch.add_argument("--timeout", type=float, default=20)
|
|
watch.add_argument("--out", help="JSON file, written only after a complete report")
|
|
watch.add_argument("command", nargs=argparse.REMAINDER)
|
|
args = parser.parse_args()
|
|
if args.mode == "check":
|
|
return check(args)
|
|
if args.mode == "retry":
|
|
return retry(args)
|
|
args.command = [c for c in args.command if c != "--"]
|
|
if not args.command:
|
|
parser.error(f"{args.mode} needs an action command after --")
|
|
return load(args) if args.mode == "load" else detect(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|