Add the k6 load profile and the load and detect probe modes

The k6 profile from the prepared kit runs an open model against the
balancer from the edge network: constant arrival rate, 80 % keyed
message posts and 20 % history reads over 20 synthetic users and 10
chats, metrics only for the measure phase, and no setup data (tokens)
in summary.json. scripts/k6.sh runs it with the host uid so the
summary is writable on Linux too.

scripts/probe.py gains three things. check --rounds repeats the
same-key race with a new key per round and counts rounds with
identical bodies, one stored copy and 409 on another body. load runs
an action under k6 while a probe posts every 0.1 s and resends the
same key after a failure and a poller reads the number of UP servers;
it writes the action log, the probe and LB traces, the delta of the
backend retry, redispatch and error counters, and a verdict: dip for
a replica that leaves and returns, replace for a rolling update that
must keep capacity and end on new replicas only. detect runs the
action in the background and polls the balancer at the same time, so
a drain that ends after the exclusion is visible, and checks the
declared maximum, the LB answer after detection and the time to
restore.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-28 19:51:59 +03:00
co-authored by Claude Opus 5.5
parent 502c7d8ace
commit 711c9c98dc
4 changed files with 446 additions and 32 deletions
+106
View File
@@ -0,0 +1,106 @@
import http from 'k6/http';
import { check } from 'k6';
import encoding from 'k6/encoding';
import exec from 'k6/execution';
const BASE = __ENV.BASE_URL || 'http://lb:8080';
const RATE = Number(__ENV.RATE || 50);
const WARMUP = Number(__ENV.WARMUP || 10);
const DURATION = Number(__ENV.DURATION || 60);
const USERS = Math.max(2, Number(__ENV.USERS || 20));
const READ_RATIO = Number(__ENV.READ_RATIO || 0.2);
const VUS = Number(__ENV.VUS || Math.max(10, Math.ceil(RATE / 10)));
const MAX_VUS = Number(__ENV.MAX_VUS || Math.max(VUS, Math.min(VUS * 4, 1000)));
const RUN = __ENV.RUN_ID || Date.now().toString(36);
const SUMMARY = __ENV.SUMMARY || '/out/summary.json';
const JSON_HEADERS = { 'Content-Type': 'application/json' };
const phase = (startTime, duration) => ({
executor: 'constant-arrival-rate',
rate: RATE,
timeUnit: '1s',
startTime: `${startTime}s`,
duration: `${duration}s`,
preAllocatedVUs: VUS,
maxVUs: MAX_VUS,
gracefulStop: '5s',
});
const measured = ['http_reqs', 'http_req_failed', 'http_req_duration', 'dropped_iterations', 'iterations'];
export const options = {
setupTimeout: '180s',
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'],
scenarios: Object.assign(WARMUP > 0 ? { warmup: phase(0, WARMUP) } : {}, { measure: phase(WARMUP, DURATION) }),
thresholds: Object.fromEntries([
...measured.map((metric) => [`${metric}{scenario:measure}`, []]),
['http_req_duration{scenario:measure,name:post_message}', []],
['http_req_duration{scenario:measure,name:get_history}', []],
]),
};
function created(response, what) {
if (response.status !== 201) {
throw new Error(`${what}: HTTP ${response.status}`);
}
return response.json();
}
const bearer = (token, extra) => Object.assign({ Authorization: `Bearer ${token}` }, extra || {});
export function setup() {
const actors = [];
for (let i = 0; i < USERS; i++) {
const username = `k6_${RUN}_${i}`.slice(0, 32);
const password = `K6-${RUN}-${i}-synthetic`;
const user = created(
http.post(`${BASE}/api/v1/users`, JSON.stringify({ username, display_name: `k6 ${i}`, password }), { headers: JSON_HEADERS }),
'register',
);
const basic = encoding.b64encode(`${username}:${password}`);
const session = created(
http.post(`${BASE}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${basic}` } }),
'login',
);
actors.push({ id: user.id, token: session.token });
}
const chats = [];
for (let i = 0; i < USERS; i += 2) {
const [a, b] = [actors[i], actors[(i + 1) % USERS]];
const body = JSON.stringify({ title: `k6 ${RUN} ${i}`, member_ids: [a.id, b.id] });
const chat = created(http.post(`${BASE}/api/v1/chats`, body, { headers: bearer(a.token, JSON_HEADERS) }), 'chat');
chats.push({ id: chat.id, tokens: [a.token, b.token] });
}
console.log('LOAD-START');
return { chats };
}
export default function (data) {
const n = exec.scenario.iterationInTest;
const chat = data.chats[n % data.chats.length];
const token = chat.tokens[Math.floor(n / data.chats.length) % 2];
if (Math.random() < READ_RATIO) {
const response = http.get(`${BASE}/api/v1/chats/${chat.id}/messages?limit=50`, {
headers: bearer(token),
tags: { name: 'get_history' },
timeout: '10s',
});
check(response, { 'history 200': (r) => r.status === 200 });
return;
}
const key = `${RUN}-${exec.scenario.name}-${n}`;
const response = http.post(`${BASE}/api/v1/chats/${chat.id}/messages`, JSON.stringify({ text: `k6 ${key}` }), {
headers: bearer(token, Object.assign({ 'Idempotency-Key': key }, JSON_HEADERS)),
tags: { name: 'post_message' },
timeout: '10s',
});
check(response, { 'message 201': (r) => r.status === 201 });
}
export function handleSummary(data) {
delete data.setup_data;
const reqs = data.metrics['http_reqs{scenario:measure}'];
const failed = data.metrics['http_req_failed{scenario:measure}'];
const line = `measure: ${reqs ? reqs.values.count : 0} requests, failed ratio ${failed ? failed.values.rate : 'n/a'}\n`;
return { [SUMMARY]: JSON.stringify(data, null, 1), stdout: line };
}
@@ -9,14 +9,14 @@
- [x] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile
- [x] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh`
- [x] 2.4 HAProxy `init-state fully-down`: вернувшаяся через DNS реплика ждёт `rise 2` проверок
- [x] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"`
- [x] 2.3 HAProxy `init-state fully-down`: вернувшаяся через DNS реплика ждёт `rise 2` проверок
- [x] 2.4 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"`
## 3. Нагрузка и зонд
- [ ] 3.1 `infra/k6/messages.js`, `scripts/k6.sh` (запуск от uid хоста)
- [ ] 3.2 `scripts/probe.py`: `load` (k6 + зонд + опрос LB, `action.log`, счётчики повторов LB, `--expect dip|replace`, `pass`), `detect` (действие фоном, `--max`, `--until`, `--within-action`, `--lb-status`, `--restore`), `check --rounds`
- [ ] 3.3 Проверка группы: `make lint`, `make check` (разбор `scripts/*.py` интерпретатором CI), `k6 inspect`
- [x] 3.1 `infra/k6/messages.js`, `scripts/k6.sh` (запуск от uid хоста)
- [x] 3.2 `scripts/probe.py`: `load` (k6 + зонд + опрос LB, `action.log`, счётчики повторов LB, `--expect dip|replace`, `pass`), `detect` (действие фоном, `--max`, `--until`, `--within-action`, `--lb-status`, `--restore`), `check --rounds`
- [x] 3.3 Проверка группы: `make lint`, `make check` (разбор `scripts/*.py` интерпретатором CI), `k6 inspect`
## 4. Сценарии уровня «4»
Executable
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
out=${1:?usage: k6.sh <output dir> [k6 run args]}
shift
mkdir -p "$out"
out=$(cd "$out" && pwd)
cd "$(dirname "$0")/.."
exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \
-e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \
k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}"
+325 -27
View File
@@ -8,12 +8,14 @@ import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
@functools.cache
@@ -27,6 +29,10 @@ def base_url():
return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080")
def stats_url():
return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv"
def call(method, path, body=None, headers=None, timeout=2.0):
data = None if body is None else json.dumps(body, ensure_ascii=False).encode()
head = {"Accept": "application/json"}
@@ -41,7 +47,11 @@ def call(method, path, body=None, headers=None, timeout=2.0):
payload = json.loads(response.read() or b"null")
return response.status, response.headers.get("X-Instance-Id", ""), payload
except urllib.error.HTTPError as error:
return error.code, error.headers.get("X-Instance-Id", ""), None
try:
payload = json.loads(error.read() or b"null")
except (OSError, ValueError):
payload = None
return error.code, error.headers.get("X-Instance-Id", ""), payload
except (OSError, ValueError):
return 0, "", None
@@ -85,53 +95,341 @@ def send(owner, chat, text, key):
return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers)
def lb_rows(backend="api"):
try:
with urllib.request.urlopen(stats_url(), timeout=2) as response:
lines = response.read().decode().splitlines()
except OSError:
return None
header = lines[0].lstrip("# ").split(",")
rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line]
return [r for r in rows if r["pxname"] == backend]
def lb_up():
rows = lb_rows()
if rows is None:
return None
return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP"))
def lb_counters():
row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {})
return {name: int(row.get(name) or 0) for name in COUNTERS}
def run_k6(out, env, on_start=None):
Path(out).mkdir(parents=True, exist_ok=True)
started = []
process = subprocess.Popen(
[str(ROOT / "scripts/k6.sh"), str(out)],
env=dict(os.environ, **env),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
with (Path(out) / "k6.log").open("w") as log:
for line in process.stdout:
log.write(line)
if not started and "LOAD-START" in line:
started.append(time.time())
if on_start:
on_start()
return process.wait(), started[0] if started else None
def k6_metrics(path, duration):
metrics = json.loads(Path(path).read_text())["metrics"]
def value(name, field):
return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0)
count = value("http_reqs", "count")
return {
"requests": int(count),
"rps": round(count / duration, 2),
"ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2),
"failed_ratio": round(value("http_req_failed", "rate"), 5),
"dropped_iterations": int(value("dropped_iterations", "count")),
"p50_ms": round(value("http_req_duration", "med"), 2),
"p95_ms": round(value("http_req_duration", "p(95)"), 2),
"p99_ms": round(value("http_req_duration", "p(99)"), 2),
}
def race(alice, chat, parallel):
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
barrier, results = threading.Barrier(parallel), []
def worker():
barrier.wait()
results.append(send(alice, chat, text, key))
threads = [threading.Thread(target=worker) for _ in range(parallel)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
bodies = [r[2] for r in results if r[0] == 201]
identical = bool(bodies) and all(b == bodies[0] for b in bodies)
return text, results, identical, send(alice, chat, text + " changed", key)[0]
def check(args):
alice, bob = actor("alice"), actor("bob")
chat = chat_of(alice, bob)
seen = collections.Counter(
call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples)
)
key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex
barrier, results = threading.Barrier(args.parallel), []
def worker():
barrier.wait()
results.append(send(alice, chat, text, key))
threads = [threading.Thread(target=worker) for _ in range(args.parallel)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))]
page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2]
bodies = [r[2] for r in results if r[0] == 201]
race = {
copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"])
statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results)
served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results)
same = {
"rounds": len(rounds),
"parallel": args.parallel,
"statuses": dict(collections.Counter(r[0] for r in results)),
"instances": dict(collections.Counter(r[1] for r in results)),
"identical_bodies": bool(bodies) and all(b == bodies[0] for b in bodies),
"stored_copies": sum(1 for m in (page or {"items": []})["items"] if m.get("text") == text),
"other_body_status": send(alice, chat, text + " changed", key)[0],
"statuses": dict(statuses),
"instances": dict(served),
"identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical),
"single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1),
"conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409),
}
passed = (
len(seen) >= 2
and race["statuses"] == {201: args.parallel}
and race["identical_bodies"]
and race["stored_copies"] == 1
and race["other_body_status"] == 409
and statuses == {201: len(rounds) * args.parallel}
and len(served) >= 2
and same["identical_bodies_rounds"] == len(rounds)
and same["single_copy_rounds"] == len(rounds)
and same["conflict_rounds"] == len(rounds)
)
emit({"instances_one_token": dict(seen), "same_key": race, "pass": passed}, args.out)
emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out)
return 0 if passed else 1
def record(stop, alice, chat):
probes, trace = [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
key = uuid.uuid4().hex
time.sleep(0.1)
def poller():
while not stop.is_set():
trace.append((time.time(), lb_up()))
time.sleep(0.1)
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
def write_csv(path, header, rows):
with path.open("w") as handle:
handle.write(header + "\n")
handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows)
def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
served = [(t, i) for t, s, _, i in probes if s == 201]
before = collections.Counter(i for t, i in served if t < t0)
during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1])
after = collections.Counter(i for t, i in served if t > end)
return {
"label": label,
"action": args.command,
"action_exit": action["exit"],
"action_seconds": round(end - t0, 3),
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
},
}
def verdict(args, result):
probe, k6 = result["probe"], result["k6"]
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0
checks["rejoined"] = result["rejoin_seconds"] is not None and bool(
probe["instances_after"].get(result["label"])
)
if args.expect == "replace":
checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"]
old, new = set(probe["instances_before"]), set(probe["instances_after"])
checks["all_replaced"] = bool(new) and not old & new
return checks
def load(args):
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
def act():
time.sleep(args.warmup + args.lead)
action["start"] = time.time()
with (out / "action.log").open("w") as log:
action["exit"] = subprocess.call(
args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT
)
action["end"] = time.time()
runner = threading.Thread(target=act)
env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)}
counters = lb_counters()
code, load_start = run_k6(out / "k6", env, runner.start)
if load_start is not None:
runner.join()
action["load_end"] = load_start + args.warmup + args.duration
stop.set()
for thread in threads:
thread.join()
if "start" not in action:
raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.log'}")
t0 = action["start"]
write_csv(
out / "probe.csv",
"t_rel_s,status,latency_ms,instance",
((round(t - t0, 3), *r) for t, *r in probes),
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
)
result["checks"] = verdict(args, result)
result["pass"] = all(result["checks"].values())
(out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result, ensure_ascii=False))
return 0 if result["pass"] else 1
def detect(args):
before = lb_up() or 0
until = before - 1 if args.until is None else args.until
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
acted = time.time() - started if acted is None else acted
status, instance, payload = call("GET", "/health/live")
restored = None
if args.restore:
begun = time.time()
subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT))
while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout:
time.sleep(0.1)
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
if args.within_action:
checks["excluded_before_action_end"] = detected is not None and detected < acted
if args.restore:
checks["restored"] = restored is not None
report = {
"action": args.command,
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
"measurement_slack_seconds": args.slack,
"lb_answer_after_detection": {
"path": "/health/live",
"status": status,
"x_instance_id_present": bool(instance),
"error_code": ((payload or {}).get("error") or {}).get("code"),
},
"restore": args.restore,
"restore_seconds": restored,
"lb_up_after_restore": lb_up(),
"checks": checks,
"pass": all(checks.values()),
}
emit(report, args.out)
return 0 if report["pass"] else 1
def main():
parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract")
sub = parser.add_subparsers(dest="mode", required=True)
one = sub.add_parser("check", help="instances behind one token and a same-key race")
one = sub.add_parser("check", help="instances behind one token and same-key races")
one.add_argument("--samples", type=int, default=40)
one.add_argument("--parallel", type=int, default=8)
one.add_argument("--rounds", type=int, default=1)
one.add_argument("--out", help="JSON file, written only after a complete report")
return check(parser.parse_args())
under = sub.add_parser("load", help="run an action under k6 load and measure recovery")
under.add_argument("--out", required=True)
under.add_argument("--label")
under.add_argument("--expect", choices=("dip", "replace"), default="dip")
under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50")))
under.add_argument("--warmup", type=int, default=10)
under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60")))
under.add_argument("--lead", type=int, default=15)
under.add_argument("--max-failed-ratio", type=float, default=0.01)
under.add_argument("--max-recovery", type=float, default=10.0)
under.add_argument("command", nargs=argparse.REMAINDER)
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
watch.add_argument("--until", type=int, help="UP servers counted as detected")
watch.add_argument("--within-action", action="store_true")
watch.add_argument("--lb-status", type=int, default=200)
watch.add_argument("--restore", help="shell command that brings replicas back")
watch.add_argument("--timeout", type=float, default=20)
watch.add_argument("--out", help="JSON file, written only after a complete report")
watch.add_argument("command", nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.mode == "check":
return check(args)
args.command = [c for c in args.command if c != "--"]
if not args.command:
parser.error(f"{args.mode} needs an action command after --")
return load(args) if args.mode == "load" else detect(args)
if __name__ == "__main__":