forked from mtusi2026/mtusi2026lab2
scripts/detection.sh runs four failures one after another and brings the stand back between them: SIGTERM through docker stop -t 6 (the balancer must exclude the replica within 1.5 s and before the process exits after its drain window), docker kill (1.5 s), docker pause for a hung process (3.5 s, the check timeout counts twice) and stopping PostgreSQL (all replicas out within 1.5 s, the balancer answers 503 itself). A 0.25 s slack covers the docker CLI and the 50 ms polling. Each run writes evidence/detection/<case>.json and the script fails if any case exceeds its declared maximum. scripts/failover-each.sh stops every replica in turn with stop-one.sh, keeps it down for 10 s and restores it, all inside one k6 load window, and collects a summary with RPS, errors, latency, detection, rejoin, probe recovery and balancer retry counters per replica. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
20 lines
720 B
Bash
Executable File
20 lines
720 B
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
out=${1:-evidence/failover-each}
|
|
rm -rf "$out"
|
|
mkdir -p "$out"
|
|
scripts/restore.sh
|
|
hosts=$(docker inspect -f '{{.Config.Hostname}}' $(docker compose ps -q "${API_SERVICE:-api}"))
|
|
failed=0
|
|
set --
|
|
for host in $hosts; do
|
|
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
|
|
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
|
|
scripts/restore.sh
|
|
set -- "$@" "$out/$host/result.json"
|
|
done
|
|
python3 -c 'import json,sys; print(json.dumps([json.load(open(p)) for p in sys.argv[1:]], ensure_ascii=False, indent=2))' "$@" > "$out/summary.tmp"
|
|
mv "$out/summary.tmp" "$out/summary.json"
|
|
exit "$failed"
|