62 lines
3.0 KiB
Bash
Executable File
62 lines
3.0 KiB
Bash
Executable File
#!/bin/sh
|
|
# Level 4: one API replica fails under a constant write load. The generator
|
|
# sends messages at RATE for DURATION and retries uncertain sends with the
|
|
# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM,
|
|
# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started
|
|
# again. Afterwards the generator reads the history back: every confirmed
|
|
# message once.
|
|
#
|
|
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
|
|
#
|
|
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35),
|
|
# OUT (default evidence/failover). Results in <OUT>/<kind>-<n>/:
|
|
# timeline.json, seconds.csv (per-second outcomes, attributed by send time),
|
|
# faults.json (when and what was stopped), compose.log (the balancer's
|
|
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
|
|
# Runs on the current stand (`make up`) and leaves every replica running.
|
|
set -eu
|
|
cd "$(dirname "$0")/../.."
|
|
. scripts/bench/lib.sh
|
|
|
|
kind=${1:?usage: failover.sh stop|kill <replica number>}
|
|
n=${2:?usage: failover.sh stop|kill <replica number>}
|
|
case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac
|
|
rate=${RATE:-50}
|
|
duration=${DURATION:-60s}
|
|
fault_at=${FAULT_AT:-15}
|
|
restore_at=${RESTORE_AT:-35}
|
|
dir=${OUT:-evidence/failover}/$kind-$n
|
|
export MSYS_NO_PATHCONV=1
|
|
|
|
rm -rf "$dir"
|
|
mkdir -p "$dir"
|
|
docker compose up -d --wait >/dev/null 2>&1
|
|
target=$(docker compose ps -q api | sed -n "${n}p")
|
|
[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; }
|
|
alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }')
|
|
meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \
|
|
"fault_at_s=$fault_at" "restore_at_s=$restore_at"
|
|
sh scripts/instances.sh >"$dir/instances-before.txt"
|
|
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
|
|
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
|
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
|
generator=$!
|
|
# The generator seeds two users first; its timeline starts after that. The
|
|
# fault times below are absolute, as is started_at in timeline.json.
|
|
sleep "$fault_at"
|
|
fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
docker "$kind" "$target" >/dev/null
|
|
fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
sleep $((restore_at - fault_at))
|
|
restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
docker compose start api >/dev/null 2>&1
|
|
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
|
|
|
|
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \
|
|
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json"
|
|
docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1
|
|
sh scripts/instances.sh >"$dir/instances-after.txt"
|
|
cat "$dir/loadgen.log" | tail -n 1
|
|
echo "results in $dir"
|