diff --git a/infra/k6/messages.js b/infra/k6/messages.js new file mode 100644 index 00000000..722cf980 --- /dev/null +++ b/infra/k6/messages.js @@ -0,0 +1,106 @@ +import http from 'k6/http'; +import { check } from 'k6'; +import encoding from 'k6/encoding'; +import exec from 'k6/execution'; + +const BASE = __ENV.BASE_URL || 'http://lb:8080'; +const RATE = Number(__ENV.RATE || 50); +const WARMUP = Number(__ENV.WARMUP || 10); +const DURATION = Number(__ENV.DURATION || 60); +const USERS = Math.max(2, Number(__ENV.USERS || 20)); +const READ_RATIO = Number(__ENV.READ_RATIO || 0.2); +const VUS = Number(__ENV.VUS || Math.max(10, Math.ceil(RATE / 10))); +const MAX_VUS = Number(__ENV.MAX_VUS || Math.max(VUS, Math.min(VUS * 4, 1000))); +const RUN = __ENV.RUN_ID || Date.now().toString(36); +const SUMMARY = __ENV.SUMMARY || '/out/summary.json'; +const JSON_HEADERS = { 'Content-Type': 'application/json' }; + +const phase = (startTime, duration) => ({ + executor: 'constant-arrival-rate', + rate: RATE, + timeUnit: '1s', + startTime: `${startTime}s`, + duration: `${duration}s`, + preAllocatedVUs: VUS, + maxVUs: MAX_VUS, + gracefulStop: '5s', +}); + +const measured = ['http_reqs', 'http_req_failed', 'http_req_duration', 'dropped_iterations', 'iterations']; + +export const options = { + setupTimeout: '180s', + summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'], + scenarios: Object.assign(WARMUP > 0 ? { warmup: phase(0, WARMUP) } : {}, { measure: phase(WARMUP, DURATION) }), + thresholds: Object.fromEntries([ + ...measured.map((metric) => [`${metric}{scenario:measure}`, []]), + ['http_req_duration{scenario:measure,name:post_message}', []], + ['http_req_duration{scenario:measure,name:get_history}', []], + ]), +}; + +function created(response, what) { + if (response.status !== 201) { + throw new Error(`${what}: HTTP ${response.status}`); + } + return response.json(); +} + +const bearer = (token, extra) => Object.assign({ Authorization: `Bearer ${token}` }, extra || {}); + +export function setup() { + const actors = []; + for (let i = 0; i < USERS; i++) { + const username = `k6_${RUN}_${i}`.slice(0, 32); + const password = `K6-${RUN}-${i}-synthetic`; + const user = created( + http.post(`${BASE}/api/v1/users`, JSON.stringify({ username, display_name: `k6 ${i}`, password }), { headers: JSON_HEADERS }), + 'register', + ); + const basic = encoding.b64encode(`${username}:${password}`); + const session = created( + http.post(`${BASE}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${basic}` } }), + 'login', + ); + actors.push({ id: user.id, token: session.token }); + } + const chats = []; + for (let i = 0; i < USERS; i += 2) { + const [a, b] = [actors[i], actors[(i + 1) % USERS]]; + const body = JSON.stringify({ title: `k6 ${RUN} ${i}`, member_ids: [a.id, b.id] }); + const chat = created(http.post(`${BASE}/api/v1/chats`, body, { headers: bearer(a.token, JSON_HEADERS) }), 'chat'); + chats.push({ id: chat.id, tokens: [a.token, b.token] }); + } + console.log('LOAD-START'); + return { chats }; +} + +export default function (data) { + const n = exec.scenario.iterationInTest; + const chat = data.chats[n % data.chats.length]; + const token = chat.tokens[Math.floor(n / data.chats.length) % 2]; + if (Math.random() < READ_RATIO) { + const response = http.get(`${BASE}/api/v1/chats/${chat.id}/messages?limit=50`, { + headers: bearer(token), + tags: { name: 'get_history' }, + timeout: '10s', + }); + check(response, { 'history 200': (r) => r.status === 200 }); + return; + } + const key = `${RUN}-${exec.scenario.name}-${n}`; + const response = http.post(`${BASE}/api/v1/chats/${chat.id}/messages`, JSON.stringify({ text: `k6 ${key}` }), { + headers: bearer(token, Object.assign({ 'Idempotency-Key': key }, JSON_HEADERS)), + tags: { name: 'post_message' }, + timeout: '10s', + }); + check(response, { 'message 201': (r) => r.status === 201 }); +} + +export function handleSummary(data) { + delete data.setup_data; + const reqs = data.metrics['http_reqs{scenario:measure}']; + const failed = data.metrics['http_req_failed{scenario:measure}']; + const line = `measure: ${reqs ? reqs.values.count : 0} requests, failed ratio ${failed ? failed.values.rate : 'n/a'}\n`; + return { [SUMMARY]: JSON.stringify(data, null, 1), stdout: line }; +} diff --git a/openspec/changes/add-lab3-failover-operations/tasks.md b/openspec/changes/add-lab3-failover-operations/tasks.md index 9a72e4b8..b001f081 100644 --- a/openspec/changes/add-lab3-failover-operations/tasks.md +++ b/openspec/changes/add-lab3-failover-operations/tasks.md @@ -9,14 +9,14 @@ - [x] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile - [x] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh` -- [x] 2.4 HAProxy `init-state fully-down`: вернувшаяся через DNS реплика ждёт `rise 2` проверок -- [x] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"` +- [x] 2.3 HAProxy `init-state fully-down`: вернувшаяся через DNS реплика ждёт `rise 2` проверок +- [x] 2.4 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"` ## 3. Нагрузка и зонд -- [ ] 3.1 `infra/k6/messages.js`, `scripts/k6.sh` (запуск от uid хоста) -- [ ] 3.2 `scripts/probe.py`: `load` (k6 + зонд + опрос LB, `action.log`, счётчики повторов LB, `--expect dip|replace`, `pass`), `detect` (действие фоном, `--max`, `--until`, `--within-action`, `--lb-status`, `--restore`), `check --rounds` -- [ ] 3.3 Проверка группы: `make lint`, `make check` (разбор `scripts/*.py` интерпретатором CI), `k6 inspect` +- [x] 3.1 `infra/k6/messages.js`, `scripts/k6.sh` (запуск от uid хоста) +- [x] 3.2 `scripts/probe.py`: `load` (k6 + зонд + опрос LB, `action.log`, счётчики повторов LB, `--expect dip|replace`, `pass`), `detect` (действие фоном, `--max`, `--until`, `--within-action`, `--lb-status`, `--restore`), `check --rounds` +- [x] 3.3 Проверка группы: `make lint`, `make check` (разбор `scripts/*.py` интерпретатором CI), `k6 inspect` ## 4. Сценарии уровня «4» diff --git a/scripts/k6.sh b/scripts/k6.sh new file mode 100755 index 00000000..01d95f29 --- /dev/null +++ b/scripts/k6.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +out=${1:?usage: k6.sh [k6 run args]} +shift +mkdir -p "$out" +out=$(cd "$out" && pwd) +cd "$(dirname "$0")/.." +exec docker compose --profile bench run --rm --no-deps -T -u "$(id -u):$(id -g)" -v "$out:/out" \ + -e RATE -e WARMUP -e DURATION -e USERS -e READ_RATIO -e VUS -e MAX_VUS -e RUN_ID \ + k6 run --quiet "$@" "/scripts/${K6_SCRIPT:-messages.js}" diff --git a/scripts/probe.py b/scripts/probe.py index 44a896bd..df6516d5 100755 --- a/scripts/probe.py +++ b/scripts/probe.py @@ -8,12 +8,14 @@ import os import subprocess import sys import threading +import time import urllib.error import urllib.request import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx") @functools.cache @@ -27,6 +29,10 @@ def base_url(): return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080") +def stats_url(): + return os.environ.get("LB_STATS_URL") or f"http://{published(8404, '127.0.0.1:8404')}/stats;csv" + + def call(method, path, body=None, headers=None, timeout=2.0): data = None if body is None else json.dumps(body, ensure_ascii=False).encode() head = {"Accept": "application/json"} @@ -41,7 +47,11 @@ def call(method, path, body=None, headers=None, timeout=2.0): payload = json.loads(response.read() or b"null") return response.status, response.headers.get("X-Instance-Id", ""), payload except urllib.error.HTTPError as error: - return error.code, error.headers.get("X-Instance-Id", ""), None + try: + payload = json.loads(error.read() or b"null") + except (OSError, ValueError): + payload = None + return error.code, error.headers.get("X-Instance-Id", ""), payload except (OSError, ValueError): return 0, "", None @@ -85,53 +95,341 @@ def send(owner, chat, text, key): return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers) +def lb_rows(backend="api"): + try: + with urllib.request.urlopen(stats_url(), timeout=2) as response: + lines = response.read().decode().splitlines() + except OSError: + return None + header = lines[0].lstrip("# ").split(",") + rows = [dict(zip(header, line.split(","))) for line in lines[1:] if line] + return [r for r in rows if r["pxname"] == backend] + + +def lb_up(): + rows = lb_rows() + if rows is None: + return None + return sum(1 for r in rows if r["svname"] != "BACKEND" and r["status"].startswith("UP")) + + +def lb_counters(): + row = next((r for r in lb_rows() or [] if r["svname"] == "BACKEND"), {}) + return {name: int(row.get(name) or 0) for name in COUNTERS} + + +def run_k6(out, env, on_start=None): + Path(out).mkdir(parents=True, exist_ok=True) + started = [] + process = subprocess.Popen( + [str(ROOT / "scripts/k6.sh"), str(out)], + env=dict(os.environ, **env), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + with (Path(out) / "k6.log").open("w") as log: + for line in process.stdout: + log.write(line) + if not started and "LOAD-START" in line: + started.append(time.time()) + if on_start: + on_start() + return process.wait(), started[0] if started else None + + +def k6_metrics(path, duration): + metrics = json.loads(Path(path).read_text())["metrics"] + + def value(name, field): + return metrics.get(name + "{scenario:measure}", {}).get("values", {}).get(field, 0.0) + + count = value("http_reqs", "count") + return { + "requests": int(count), + "rps": round(count / duration, 2), + "ok_rps": round(count * (1 - value("http_req_failed", "rate")) / duration, 2), + "failed_ratio": round(value("http_req_failed", "rate"), 5), + "dropped_iterations": int(value("dropped_iterations", "count")), + "p50_ms": round(value("http_req_duration", "med"), 2), + "p95_ms": round(value("http_req_duration", "p(95)"), 2), + "p99_ms": round(value("http_req_duration", "p(99)"), 2), + } + + +def race(alice, chat, parallel): + key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex + barrier, results = threading.Barrier(parallel), [] + + def worker(): + barrier.wait() + results.append(send(alice, chat, text, key)) + + threads = [threading.Thread(target=worker) for _ in range(parallel)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + bodies = [r[2] for r in results if r[0] == 201] + identical = bool(bodies) and all(b == bodies[0] for b in bodies) + return text, results, identical, send(alice, chat, text + " changed", key)[0] + + def check(args): alice, bob = actor("alice"), actor("bob") chat = chat_of(alice, bob) seen = collections.Counter( call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples) ) - key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex - barrier, results = threading.Barrier(args.parallel), [] - - def worker(): - barrier.wait() - results.append(send(alice, chat, text, key)) - - threads = [threading.Thread(target=worker) for _ in range(args.parallel)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() + rounds = [race(alice, chat, args.parallel) for _ in range(min(args.rounds, 100))] page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2] - bodies = [r[2] for r in results if r[0] == 201] - race = { + copies = collections.Counter(m.get("text") for m in (page or {"items": []})["items"]) + statuses = collections.Counter(r[0] for _, results, _, _ in rounds for r in results) + served = collections.Counter(r[1] for _, results, _, _ in rounds for r in results) + same = { + "rounds": len(rounds), "parallel": args.parallel, - "statuses": dict(collections.Counter(r[0] for r in results)), - "instances": dict(collections.Counter(r[1] for r in results)), - "identical_bodies": bool(bodies) and all(b == bodies[0] for b in bodies), - "stored_copies": sum(1 for m in (page or {"items": []})["items"] if m.get("text") == text), - "other_body_status": send(alice, chat, text + " changed", key)[0], + "statuses": dict(statuses), + "instances": dict(served), + "identical_bodies_rounds": sum(1 for _, _, identical, _ in rounds if identical), + "single_copy_rounds": sum(1 for text, _, _, _ in rounds if copies[text] == 1), + "conflict_rounds": sum(1 for _, _, _, other in rounds if other == 409), } passed = ( len(seen) >= 2 - and race["statuses"] == {201: args.parallel} - and race["identical_bodies"] - and race["stored_copies"] == 1 - and race["other_body_status"] == 409 + and statuses == {201: len(rounds) * args.parallel} + and len(served) >= 2 + and same["identical_bodies_rounds"] == len(rounds) + and same["single_copy_rounds"] == len(rounds) + and same["conflict_rounds"] == len(rounds) ) - emit({"instances_one_token": dict(seen), "same_key": race, "pass": passed}, args.out) + emit({"instances_one_token": dict(seen), "same_key": same, "pass": passed}, args.out) return 0 if passed else 1 +def record(stop, alice, chat): + probes, trace = [], [] + + def prober(): + key = uuid.uuid4().hex + while not stop.is_set(): + started = time.time() + status, instance, _ = send(alice, chat, "probe " + key, key) + probes.append((started, status, round(1000 * (time.time() - started), 1), instance)) + if status == 201: + key = uuid.uuid4().hex + time.sleep(0.1) + + def poller(): + while not stop.is_set(): + trace.append((time.time(), lb_up())) + time.sleep(0.1) + + threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)] + for thread in threads: + thread.start() + return probes, trace, threads + + +def write_csv(path, header, rows): + with path.open("w") as handle: + handle.write(header + "\n") + handle.writelines(",".join(str(v) for v in row) + "\n" for row in rows) + + +def summarize(args, probes, trace, action, expected): + t0, end = action["start"], action["end"] + label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action") + failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0] + down = [t for t, up in trace if t >= t0 and up is not None and up < expected] + back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected] + outage = [t for t in down if not back or t < back[0]] or [end] + served = [(t, i) for t, s, _, i in probes if s == 201] + before = collections.Counter(i for t, i in served if t < t0) + during = collections.Counter(i for t, i in served if outage[0] <= t <= outage[-1]) + after = collections.Counter(i for t, i in served if t > end) + return { + "label": label, + "action": args.command, + "action_exit": action["exit"], + "action_seconds": round(end - t0, 3), + "action_finished_before_load_end": end < action["load_end"], + "lb_up_before": expected, + "lb_up_min": min((up for _, up in trace if up is not None), default=None), + "detection_seconds": round(down[0] - t0, 3) if down else None, + "rejoin_seconds": round(back[0] - t0, 3) if back else None, + "probe": { + "attempts": len(probes), + "failed_attempts": sum(1 for _, s, _, _ in probes if s != 201), + "first_failure_s": round(failures[0], 3) if failures else None, + "recovery_s": round(failures[-1], 3) if failures else 0.0, + "instances_before": dict(before), + "instances_during_outage": dict(during), + "instances_after": dict(after), + }, + } + + +def verdict(args, result): + probe, k6 = result["probe"], result["k6"] + checks = { + "action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"], + "load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio, + "write_recovery_ok": probe["recovery_s"] <= args.max_recovery, + } + if args.expect == "dip": + checks["detected"] = result["detection_seconds"] is not None + checks["target_excluded"] = probe["instances_during_outage"].get(result["label"], 0) == 0 + checks["rejoined"] = result["rejoin_seconds"] is not None and bool( + probe["instances_after"].get(result["label"]) + ) + if args.expect == "replace": + checks["capacity_kept"] = (result["lb_up_min"] or 0) >= result["lb_up_before"] + old, new = set(probe["instances_before"]), set(probe["instances_after"]) + checks["all_replaced"] = bool(new) and not old & new + return checks + + +def load(args): + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + expected = lb_up() or 0 + alice, bob = actor("alice"), actor("bob") + stop, action = threading.Event(), {} + probes, trace, threads = record(stop, alice, chat_of(alice, bob)) + + def act(): + time.sleep(args.warmup + args.lead) + action["start"] = time.time() + with (out / "action.log").open("w") as log: + action["exit"] = subprocess.call( + args.command, cwd=str(ROOT), stdout=log, stderr=subprocess.STDOUT + ) + action["end"] = time.time() + + runner = threading.Thread(target=act) + env = {"RATE": str(args.rate), "WARMUP": str(args.warmup), "DURATION": str(args.duration)} + counters = lb_counters() + code, load_start = run_k6(out / "k6", env, runner.start) + if load_start is not None: + runner.join() + action["load_end"] = load_start + args.warmup + args.duration + stop.set() + for thread in threads: + thread.join() + if "start" not in action: + raise SystemExit(f"k6 failed before the action (exit {code}), see {out / 'k6/k6.log'}") + t0 = action["start"] + write_csv( + out / "probe.csv", + "t_rel_s,status,latency_ms,instance", + ((round(t - t0, 3), *r) for t, *r in probes), + ) + write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace)) + result = summarize(args, probes, trace, action, expected) + result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()} + result["k6"] = dict( + k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate + ) + result["checks"] = verdict(args, result) + result["pass"] = all(result["checks"].values()) + (out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(result, ensure_ascii=False)) + return 0 if result["pass"] else 1 + + +def detect(args): + before = lb_up() or 0 + until = before - 1 if args.until is None else args.until + started = time.time() + process = subprocess.Popen(args.command, cwd=str(ROOT)) + detected = acted = None + while time.time() - started < args.timeout and (detected is None or acted is None): + if acted is None and process.poll() is not None: + acted = time.time() - started + if detected is None and (lb_up() or 0) <= until: + detected = time.time() - started + time.sleep(0.05) + code = process.wait() + acted = time.time() - started if acted is None else acted + status, instance, payload = call("GET", "/health/live") + restored = None + if args.restore: + begun = time.time() + subprocess.call(["sh", "-c", args.restore], cwd=str(ROOT)) + while (lb_up() or 0) < before and time.time() - begun < 3 * args.timeout: + time.sleep(0.1) + restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None + checks = { + "action_ok": code == 0, + "detected_in_time": detected is not None and detected <= args.max + args.slack, + "lb_answer_ok": status == args.lb_status, + } + if args.within_action: + checks["excluded_before_action_end"] = detected is not None and detected < acted + if args.restore: + checks["restored"] = restored is not None + report = { + "action": args.command, + "action_exit": code, + "action_seconds": round(acted, 3), + "lb_up_before": before, + "lb_up_counted_as_detected": until, + "detection_seconds": None if detected is None else round(detected, 3), + "declared_max_seconds": args.max, + "measurement_slack_seconds": args.slack, + "lb_answer_after_detection": { + "path": "/health/live", + "status": status, + "x_instance_id_present": bool(instance), + "error_code": ((payload or {}).get("error") or {}).get("code"), + }, + "restore": args.restore, + "restore_seconds": restored, + "lb_up_after_restore": lb_up(), + "checks": checks, + "pass": all(checks.values()), + } + emit(report, args.out) + return 0 if report["pass"] else 1 + + def main(): parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract") sub = parser.add_subparsers(dest="mode", required=True) - one = sub.add_parser("check", help="instances behind one token and a same-key race") + one = sub.add_parser("check", help="instances behind one token and same-key races") one.add_argument("--samples", type=int, default=40) one.add_argument("--parallel", type=int, default=8) + one.add_argument("--rounds", type=int, default=1) one.add_argument("--out", help="JSON file, written only after a complete report") - return check(parser.parse_args()) + under = sub.add_parser("load", help="run an action under k6 load and measure recovery") + under.add_argument("--out", required=True) + under.add_argument("--label") + under.add_argument("--expect", choices=("dip", "replace"), default="dip") + under.add_argument("--rate", type=int, default=int(os.environ.get("RATE", "50"))) + under.add_argument("--warmup", type=int, default=10) + under.add_argument("--duration", type=int, default=int(os.environ.get("DURATION", "60"))) + under.add_argument("--lead", type=int, default=15) + under.add_argument("--max-failed-ratio", type=float, default=0.01) + under.add_argument("--max-recovery", type=float, default=10.0) + under.add_argument("command", nargs=argparse.REMAINDER) + watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation") + watch.add_argument("--max", type=float, required=True, help="declared worst detection, s") + watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s") + watch.add_argument("--until", type=int, help="UP servers counted as detected") + watch.add_argument("--within-action", action="store_true") + watch.add_argument("--lb-status", type=int, default=200) + watch.add_argument("--restore", help="shell command that brings replicas back") + watch.add_argument("--timeout", type=float, default=20) + watch.add_argument("--out", help="JSON file, written only after a complete report") + watch.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.mode == "check": + return check(args) + args.command = [c for c in args.command if c != "--"] + if not args.command: + parser.error(f"{args.mode} needs an action command after --") + return load(args) if args.mode == "load" else detect(args) if __name__ == "__main__":