Add the k6 load profile and the load and detect probe modes
The k6 profile from the prepared kit runs an open model against the balancer from the edge network: constant arrival rate, 80 % keyed message posts and 20 % history reads over 20 synthetic users and 10 chats, metrics only for the measure phase, and no setup data (tokens) in summary.json. scripts/k6.sh runs it with the host uid so the summary is writable on Linux too. scripts/probe.py gains three things. check --rounds repeats the same-key race with a new key per round and counts rounds with identical bodies, one stored copy and 409 on another body. load runs an action under k6 while a probe posts every 0.1 s and resends the same key after a failure and a poller reads the number of UP servers; it writes the action log, the probe and LB traces, the delta of the backend retry, redispatch and error counters, and a verdict: dip for a replica that leaves and returns, replace for a rolling update that must keep capacity and end on new replicas only. detect runs the action in the background and polls the balancer at the same time, so a drain that ends after the exclusion is visible, and checks the declared maximum, the LB answer after detection and the time to restore. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
502c7d8ace
commit
711c9c98dc
@@ -0,0 +1,106 @@
|
||||
import http from 'k6/http';
|
||||
import { check } from 'k6';
|
||||
import encoding from 'k6/encoding';
|
||||
import exec from 'k6/execution';
|
||||
|
||||
const BASE = __ENV.BASE_URL || 'http://lb:8080';
|
||||
const RATE = Number(__ENV.RATE || 50);
|
||||
const WARMUP = Number(__ENV.WARMUP || 10);
|
||||
const DURATION = Number(__ENV.DURATION || 60);
|
||||
const USERS = Math.max(2, Number(__ENV.USERS || 20));
|
||||
const READ_RATIO = Number(__ENV.READ_RATIO || 0.2);
|
||||
const VUS = Number(__ENV.VUS || Math.max(10, Math.ceil(RATE / 10)));
|
||||
const MAX_VUS = Number(__ENV.MAX_VUS || Math.max(VUS, Math.min(VUS * 4, 1000)));
|
||||
const RUN = __ENV.RUN_ID || Date.now().toString(36);
|
||||
const SUMMARY = __ENV.SUMMARY || '/out/summary.json';
|
||||
const JSON_HEADERS = { 'Content-Type': 'application/json' };
|
||||
|
||||
const phase = (startTime, duration) => ({
|
||||
executor: 'constant-arrival-rate',
|
||||
rate: RATE,
|
||||
timeUnit: '1s',
|
||||
startTime: `${startTime}s`,
|
||||
duration: `${duration}s`,
|
||||
preAllocatedVUs: VUS,
|
||||
maxVUs: MAX_VUS,
|
||||
gracefulStop: '5s',
|
||||
});
|
||||
|
||||
const measured = ['http_reqs', 'http_req_failed', 'http_req_duration', 'dropped_iterations', 'iterations'];
|
||||
|
||||
export const options = {
|
||||
setupTimeout: '180s',
|
||||
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'],
|
||||
scenarios: Object.assign(WARMUP > 0 ? { warmup: phase(0, WARMUP) } : {}, { measure: phase(WARMUP, DURATION) }),
|
||||
thresholds: Object.fromEntries([
|
||||
...measured.map((metric) => [`${metric}{scenario:measure}`, []]),
|
||||
['http_req_duration{scenario:measure,name:post_message}', []],
|
||||
['http_req_duration{scenario:measure,name:get_history}', []],
|
||||
]),
|
||||
};
|
||||
|
||||
function created(response, what) {
|
||||
if (response.status !== 201) {
|
||||
throw new Error(`${what}: HTTP ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
const bearer = (token, extra) => Object.assign({ Authorization: `Bearer ${token}` }, extra || {});
|
||||
|
||||
export function setup() {
|
||||
const actors = [];
|
||||
for (let i = 0; i < USERS; i++) {
|
||||
const username = `k6_${RUN}_${i}`.slice(0, 32);
|
||||
const password = `K6-${RUN}-${i}-synthetic`;
|
||||
const user = created(
|
||||
http.post(`${BASE}/api/v1/users`, JSON.stringify({ username, display_name: `k6 ${i}`, password }), { headers: JSON_HEADERS }),
|
||||
'register',
|
||||
);
|
||||
const basic = encoding.b64encode(`${username}:${password}`);
|
||||
const session = created(
|
||||
http.post(`${BASE}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${basic}` } }),
|
||||
'login',
|
||||
);
|
||||
actors.push({ id: user.id, token: session.token });
|
||||
}
|
||||
const chats = [];
|
||||
for (let i = 0; i < USERS; i += 2) {
|
||||
const [a, b] = [actors[i], actors[(i + 1) % USERS]];
|
||||
const body = JSON.stringify({ title: `k6 ${RUN} ${i}`, member_ids: [a.id, b.id] });
|
||||
const chat = created(http.post(`${BASE}/api/v1/chats`, body, { headers: bearer(a.token, JSON_HEADERS) }), 'chat');
|
||||
chats.push({ id: chat.id, tokens: [a.token, b.token] });
|
||||
}
|
||||
console.log('LOAD-START');
|
||||
return { chats };
|
||||
}
|
||||
|
||||
export default function (data) {
|
||||
const n = exec.scenario.iterationInTest;
|
||||
const chat = data.chats[n % data.chats.length];
|
||||
const token = chat.tokens[Math.floor(n / data.chats.length) % 2];
|
||||
if (Math.random() < READ_RATIO) {
|
||||
const response = http.get(`${BASE}/api/v1/chats/${chat.id}/messages?limit=50`, {
|
||||
headers: bearer(token),
|
||||
tags: { name: 'get_history' },
|
||||
timeout: '10s',
|
||||
});
|
||||
check(response, { 'history 200': (r) => r.status === 200 });
|
||||
return;
|
||||
}
|
||||
const key = `${RUN}-${exec.scenario.name}-${n}`;
|
||||
const response = http.post(`${BASE}/api/v1/chats/${chat.id}/messages`, JSON.stringify({ text: `k6 ${key}` }), {
|
||||
headers: bearer(token, Object.assign({ 'Idempotency-Key': key }, JSON_HEADERS)),
|
||||
tags: { name: 'post_message' },
|
||||
timeout: '10s',
|
||||
});
|
||||
check(response, { 'message 201': (r) => r.status === 201 });
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
delete data.setup_data;
|
||||
const reqs = data.metrics['http_reqs{scenario:measure}'];
|
||||
const failed = data.metrics['http_req_failed{scenario:measure}'];
|
||||
const line = `measure: ${reqs ? reqs.values.count : 0} requests, failed ratio ${failed ? failed.values.rate : 'n/a'}\n`;
|
||||
return { [SUMMARY]: JSON.stringify(data, null, 1), stdout: line };
|
||||
}
|
||||
@@ -9,14 +9,14 @@
|
||||
|
||||
- [x] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile
|
||||
- [x] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh`
|
||||
- [x] 2.4 HAProxy `init-state fully-down`: вернувшаяся через DNS реплика ждёт `rise 2` проверок
|
||||
- [x] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"`
|
||||
- [x] 2.3 HAProxy `init-state fully-down`: вернувшаяся через DNS реплика ждёт `rise 2` проверок
|
||||
- [x] 2.4 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"`
|
||||
|
||||
## 3. Нагрузка и зонд
|
||||
|
||||
- [ ] 3.1 `infra/k6/messages.js`, `scripts/k6.sh` (запуск от uid хоста)
|
||||
- [ ] 3.2 `scripts/probe.py`: `load` (k6 + зонд + опрос LB, `action.log`, счётчики повторов LB, `--expect dip|replace`, `pass`), `detect` (действие фоном, `--max`, `--until`, `--within-action`, `--lb-status`, `--restore`), `check --rounds`
|
||||
- [ ] 3.3 Проверка группы: `make lint`, `make check` (разбор `scripts/*.py` интерпретатором CI), `k6 inspect`
|
||||
- [x] 3.1 `infra/k6/messages.js`, `scripts/k6.sh` (запуск от uid хоста)
|
||||
- [x] 3.2 `scripts/probe.py`: `load` (k6 + зонд + опрос LB, `action.log`, счётчики повторов LB, `--expect dip|replace`, `pass`), `detect` (действие фоном, `--max`, `--until`, `--within-action`, `--lb-status`, `--restore`), `check --rounds`
|
||||
- [x] 3.3 Проверка группы: `make lint`, `make check` (разбор `scripts/*.py` интерпретатором CI), `k6 inspect`
|
||||
|
||||
## 4. Сценарии уровня «4»
|
||||
|
||||
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: k6.sh <output dir> [k6 run args]}
|
||||
shift
|
||||
mkdir -p "$out"
|
||||
out=$(cd "$out" && pwd)
|
||||
cd "$(dirname "$0")/.."
|
||||
exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \
|
||||
-e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \
|
||||
k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}"
|
||||
+325
-27
@@ -8,12 +8,14 @@ import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -27,6 +29,10 @@ def base_url():
|
||||
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
|
||||
|
||||
|
||||
def stats_url():
|
||||
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, timeout=2.0):
|
||||
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
|
||||
head = {"Accept": "application/json"}
|
||||
@@ -41,7 +47,11 @@ def call(method, path, body=None, headers=None, timeout=2.0):
|
||||
payload = json.loads(response.read() or b"null")
|
||||
return response.status, response.headers.get("X-Instance-Id", ""), payload
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.headers.get("X-Instance-Id", ""), None
|
||||
try:
|
||||
payload = json.loads(error.read() or b"null")
|
||||
except (OSError, ValueError):
|
||||
payload = None
|
||||
return error.code, error.headers.get("X-Instance-Id", ""), payload
|
||||
except (OSError, ValueError):
|
||||
return 0, "", None
|
||||
|
||||
@@ -85,53 +95,341 @@ def send(owner, chat, text, key):
|
||||
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
|
||||
|
||||
|
||||
def lb_rows(backend="api"):
|
||||
try:
|
||||
with urllib.request.urlopen(stats_url(), timeout=2) as response:
|
||||
lines = response.read().decode().splitlines()
|
||||
except OSError:
|
||||
return None
|
||||
header = lines[0].lstrip("# ").split(",")
|
||||
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
|
||||
return [r for r in rows if r["pxname"] == backend]
|
||||
|
||||
|
||||
def lb_up():
|
||||
rows = lb_rows()
|
||||
if rows is None:
|
||||
return None
|
||||
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
|
||||
|
||||
|
||||
def lb_counters():
|
||||
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
|
||||
return {name: int(row.get(name) or 0) for name in COUNTERS}
|
||||
|
||||
|
||||
def run_k6(out, env, on_start=None):
|
||||
Path(out).mkdir(parents=True, exist_ok=True)
|
||||
started = []
|
||||
process = subprocess.Popen(
|
||||
[str(ROOT / "scripts/k6.sh"), str(out)],
|
||||
env=dict(os.environ, **env),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
with (Path(out) / "k6.log").open("w") as log:
|
||||
for line in process.stdout:
|
||||
log.write(line)
|
||||
if not started and "LOAD-START" in line:
|
||||
started.append(time.time())
|
||||
if on_start:
|
||||
on_start()
|
||||
return process.wait(), started[0] if started else None
|
||||
|
||||
|
||||
def k6_metrics(path, duration):
|
||||
metrics = json.loads(Path(path).read_text())["metrics"]
|
||||
|
||||
def value(name, field):
|
||||
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
|
||||
|
||||
count = value("http_reqs", "count")
|
||||
return {
|
||||
"requests": int(count),
|
||||
"rps": round(count / duration, 2),
|
||||
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
|
||||
"failed_ratio": round(value("http_req_failed", "rate"), 5),
|
||||
"dropped_iterations": int(value("dropped_iterations", "count")),
|
||||
"p50_ms": round(value("http_req_duration", "med"), 2),
|
||||
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
|
||||
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
|
||||
}
|
||||
|
||||
|
||||
def race(alice, chat, parallel):
|
||||
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
||||
barrier, results = threading.Barrier(parallel), []
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
results.append(send(alice, chat, text, key))
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(parallel)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
bodies = [r[2] for r in results if r[0] == 201]
|
||||
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
|
||||
return text, results, identical, send(alice, chat, text + " changed", key)[0]
|
||||
|
||||
|
||||
def check(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
seen = collections.Counter(
|
||||
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
|
||||
)
|
||||
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
|
||||
barrier, results = threading.Barrier(args.parallel), []
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
results.append(send(alice, chat, text, key))
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(args.parallel)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
|
||||
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
|
||||
bodies = [r[2] for r in results if r[0] == 201]
|
||||
race = {
|
||||
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
|
||||
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
|
||||
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
|
||||
same = {
|
||||
"rounds": len(rounds),
|
||||
"parallel": args.parallel,
|
||||
"statuses": dict(collections.Counter(r[0] for r in results)),
|
||||
"instances": dict(collections.Counter(r[1] for r in results)),
|
||||
"identical_bodies": bool(bodies) and all(b == bodies[0] for b in bodies),
|
||||
"stored_copies": sum(1 for m in (page or {"items": []})["items"] if m.get("text") == text),
|
||||
"other_body_status": send(alice, chat, text + " changed", key)[0],
|
||||
"statuses": dict(statuses),
|
||||
"instances": dict(served),
|
||||
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
|
||||
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
|
||||
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
|
||||
}
|
||||
passed = (
|
||||
len(seen) >= 2
|
||||
and race["statuses"] == {201: args.parallel}
|
||||
and race["identical_bodies"]
|
||||
and race["stored_copies"] == 1
|
||||
and race["other_body_status"] == 409
|
||||
and statuses == {201: len(rounds) * args.parallel}
|
||||
and len(served) >= 2
|
||||
and same["identical_bodies_rounds"] == len(rounds)
|
||||
and same["single_copy_rounds"] == len(rounds)
|
||||
and same["conflict_rounds"] == len(rounds)
|
||||
)
|
||||
emit({"instances_one_token": dict(seen), "same_key": race, "pass": passed}, args.out)
|
||||
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
|
||||
def prober():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.time()
|
||||
status, instance, _ = send(alice, chat, "probe " + key, key)
|
||||
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
||||
if status == 201:
|
||||
key = uuid.uuid4().hex
|
||||
time.sleep(0.1)
|
||||
|
||||
def poller():
|
||||
while not stop.is_set():
|
||||
trace.append((time.time(), lb_up()))
|
||||
time.sleep(0.1)
|
||||
|
||||
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
return probes, trace, threads
|
||||
|
||||
|
||||
def write_csv(path, header, rows):
|
||||
with path.open("w") as handle:
|
||||
handle.write(header + "\n")
|
||||
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
|
||||
|
||||
|
||||
def summarize(args, probes, trace, action, expected):
|
||||
t0, end = action["start"], action["end"]
|
||||
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
||||
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
||||
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
||||
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
||||
outage = [t for t in down if not back or t < back[0]] or [end]
|
||||
served = [(t, i) for t, s, _, i in probes if s == 201]
|
||||
before = collections.Counter(i for t, i in served if t < t0)
|
||||
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
|
||||
after = collections.Counter(i for t, i in served if t > end)
|
||||
return {
|
||||
"label": label,
|
||||
"action": args.command,
|
||||
"action_exit": action["exit"],
|
||||
"action_seconds": round(end - t0, 3),
|
||||
"action_finished_before_load_end": end < action["load_end"],
|
||||
"lb_up_before": expected,
|
||||
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
||||
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
||||
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
||||
"probe": {
|
||||
"attempts": len(probes),
|
||||
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
||||
"first_failure_s": round(failures[0], 3) if failures else None,
|
||||
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
||||
"instances_before": dict(before),
|
||||
"instances_during_outage": dict(during),
|
||||
"instances_after": dict(after),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def verdict(args, result):
|
||||
probe, k6 = result["probe"], result["k6"]
|
||||
checks = {
|
||||
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
||||
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
||||
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
||||
}
|
||||
if args.expect == "dip":
|
||||
checks["detected"] = result["detection_seconds"] is not None
|
||||
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
|
||||
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
|
||||
probe["instances_after"].get(result["label"])
|
||||
)
|
||||
if args.expect == "replace":
|
||||
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
|
||||
old, new = set(probe["instances_before"]), set(probe["instances_after"])
|
||||
checks["all_replaced"] = bool(new) and not old & new
|
||||
return checks
|
||||
|
||||
|
||||
def load(args):
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
expected = lb_up() or 0
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
stop, action = threading.Event(), {}
|
||||
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
||||
|
||||
def act():
|
||||
time.sleep(args.warmup + args.lead)
|
||||
action["start"] = time.time()
|
||||
with (out / "action.log").open("w") as log:
|
||||
action["exit"] = subprocess.call(
|
||||
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
|
||||
)
|
||||
action["end"] = time.time()
|
||||
|
||||
runner = threading.Thread(target=act)
|
||||
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
|
||||
counters = lb_counters()
|
||||
code, load_start = run_k6(out / "k6", env, runner.start)
|
||||
if load_start is not None:
|
||||
runner.join()
|
||||
action["load_end"] = load_start + args.warmup + args.duration
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
if "start" not in action:
|
||||
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.log'}")
|
||||
t0 = action["start"]
|
||||
write_csv(
|
||||
out / "probe.csv",
|
||||
"t_rel_s,status,latency_ms,instance",
|
||||
((round(t - t0, 3), *r) for t, *r in probes),
|
||||
)
|
||||
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
||||
result = summarize(args, probes, trace, action, expected)
|
||||
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
||||
result["k6"] = dict(
|
||||
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
||||
)
|
||||
result["checks"] = verdict(args, result)
|
||||
result["pass"] = all(result["checks"].values())
|
||||
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0 if result["pass"] else 1
|
||||
|
||||
|
||||
def detect(args):
|
||||
before = lb_up() or 0
|
||||
until = before - 1 if args.until is None else args.until
|
||||
started = time.time()
|
||||
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
||||
detected = acted = None
|
||||
while time.time() - started < args.timeout and (detected is None or acted is None):
|
||||
if acted is None and process.poll() is not None:
|
||||
acted = time.time() - started
|
||||
if detected is None and (lb_up() or 0) <= until:
|
||||
detected = time.time() - started
|
||||
time.sleep(0.05)
|
||||
code = process.wait()
|
||||
acted = time.time() - started if acted is None else acted
|
||||
status, instance, payload = call("GET", "/health/live")
|
||||
restored = None
|
||||
if args.restore:
|
||||
begun = time.time()
|
||||
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
|
||||
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
|
||||
time.sleep(0.1)
|
||||
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
||||
checks = {
|
||||
"action_ok": code == 0,
|
||||
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
||||
"lb_answer_ok": status == args.lb_status,
|
||||
}
|
||||
if args.within_action:
|
||||
checks["excluded_before_action_end"] = detected is not None and detected < acted
|
||||
if args.restore:
|
||||
checks["restored"] = restored is not None
|
||||
report = {
|
||||
"action": args.command,
|
||||
"action_exit": code,
|
||||
"action_seconds": round(acted, 3),
|
||||
"lb_up_before": before,
|
||||
"lb_up_counted_as_detected": until,
|
||||
"detection_seconds": None if detected is None else round(detected, 3),
|
||||
"declared_max_seconds": args.max,
|
||||
"measurement_slack_seconds": args.slack,
|
||||
"lb_answer_after_detection": {
|
||||
"path": "/health/live",
|
||||
"status": status,
|
||||
"x_instance_id_present": bool(instance),
|
||||
"error_code": ((payload or {}).get("error") or {}).get("code"),
|
||||
},
|
||||
"restore": args.restore,
|
||||
"restore_seconds": restored,
|
||||
"lb_up_after_restore": lb_up(),
|
||||
"checks": checks,
|
||||
"pass": all(checks.values()),
|
||||
}
|
||||
emit(report, args.out)
|
||||
return 0 if report["pass"] else 1
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
one = sub.add_parser("check", help="instances behind one token and a same-key race")
|
||||
one = sub.add_parser("check", help="instances behind one token and same-key races")
|
||||
one.add_argument("--samples", type=int, default=40)
|
||||
one.add_argument("--parallel", type=int, default=8)
|
||||
one.add_argument("--rounds", type=int, default=1)
|
||||
one.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
return check(parser.parse_args())
|
||||
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
|
||||
under.add_argument("--out", required=True)
|
||||
under.add_argument("--label")
|
||||
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
|
||||
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
|
||||
under.add_argument("--warmup", type=int, default=10)
|
||||
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
|
||||
under.add_argument("--lead", type=int, default=15)
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
watch.add_argument("--until", type=int, help="UP servers counted as detected")
|
||||
watch.add_argument("--within-action", action="store_true")
|
||||
watch.add_argument("--lb-status", type=int, default=200)
|
||||
watch.add_argument("--restore", help="shell command that brings replicas back")
|
||||
watch.add_argument("--timeout", type=float, default=20)
|
||||
watch.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
return load(args) if args.mode == "load" else detect(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user