From d5f8a7c25cd980049822b2536334f843bfb93bb8 Mon Sep 17 00:00:00 2001 From: Artyom Date: Wed, 28 Oct 2026 19:52:09 +0300 Subject: [PATCH] Measure detection and fail every replica in turn under load scripts/detection.sh runs four failures one after another and brings the stand back between them: SIGTERM through docker stop -t 6 (the balancer must exclude the replica within 1.5 s and before the process exits after its drain window), docker kill (1.5 s), docker pause for a hung process (3.5 s, the check timeout counts twice) and stopping PostgreSQL (all replicas out within 1.5 s, the balancer answers 503 itself). A 0.25 s slack covers the docker CLI and the 50 ms polling. Each run writes evidence/detection/.json and the script fails if any case exceeds its declared maximum. scripts/failover-each.sh stops every replica in turn with stop-one.sh, keeps it down for 10 s and restores it, all inside one k6 load window, and collects a summary with RPS, errors, latency, detection, rejoin, probe recovery and balancer retry counters per replica. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/detection.sh | 20 ++++++++++++++++++++ scripts/failover-each.sh | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100755 scripts/detection.sh create mode 100755 scripts/failover-each.sh diff --git a/scripts/detection.sh b/scripts/detection.sh new file mode 100755 index 00000000..5ef0d35a --- /dev/null +++ b/scripts/detection.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +out=${1:-evidence/detection} +mkdir -p "$out" +scripts/restore.sh +replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; } +detect() { python3 scripts/probe.py detect "$@"; } +failed=0 +detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \ + -- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1 +detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \ + -- docker kill "$(replica)" || failed=1 +hung=$(replica) +detect --max 3.5 --restore "docker unpause $hung && scripts/lb-wait.sh" --out "$out/pause.json" \ + -- docker pause "$hung" || failed=1 +db=$(docker compose ps -q postgres) +detect --max 1.5 --until 0 --lb-status 503 --restore "docker start $db && scripts/lb-wait.sh" \ + --out "$out/postgres.json" -- docker stop "$db" || failed=1 +exit "$failed" diff --git a/scripts/failover-each.sh b/scripts/failover-each.sh new file mode 100755 index 00000000..64050ef7 --- /dev/null +++ b/scripts/failover-each.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +out=${1:-evidence/failover-each} +rm -rf "$out" +mkdir -p "$out" +scripts/restore.sh +hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}")) +failed=0 +set -- +for host in $hosts; do + TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \ + -- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1 + scripts/restore.sh + set -- "$@" "$out/$host/result.json" +done +python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp" +mv "$out/summary.tmp" "$out/summary.json" +exit "$failed"