71 lines
3.6 KiB
Bash
Executable File
71 lines
3.6 KiB
Bash
Executable File
#!/bin/sh
|
|
# Level 4: one API replica fails under a constant write load. The generator
|
|
# sends messages at RATE for DURATION and retries uncertain sends with the
|
|
# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM,
|
|
# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started
|
|
# again. Afterwards the generator reads the history back: every confirmed
|
|
# message once.
|
|
#
|
|
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
|
|
#
|
|
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35),
|
|
# OUT (default evidence/failover). Results in <OUT>/<kind>-<n>/:
|
|
# timeline.json, seconds.csv (per-second outcomes, attributed by send time),
|
|
# faults.json (when and what was stopped), compose.log (the balancer's
|
|
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
|
|
# Runs on the current stand (`make up`) and leaves every replica running.
|
|
#
|
|
# Every time used for the analysis comes from the Docker VM's clock: the log
|
|
# timestamps, the container's stop and start times from `docker inspect`, and
|
|
# the generator's own times. The host clock of Windows runs apart from it by
|
|
# a fraction of a second, a large share of a 1.5 s detection window; host
|
|
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
|
|
# is the replica's own "shutdown started" line in compose.log.
|
|
set -eu
|
|
cd "$(dirname "$0")/../.."
|
|
. scripts/bench/lib.sh
|
|
|
|
kind=${1:?usage: failover.sh stop|kill <replica number>}
|
|
n=${2:?usage: failover.sh stop|kill <replica number>}
|
|
case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac
|
|
rate=${RATE:-50}
|
|
duration=${DURATION:-60s}
|
|
fault_at=${FAULT_AT:-15}
|
|
restore_at=${RESTORE_AT:-35}
|
|
dir=${OUT:-evidence/failover}/$kind-$n
|
|
export MSYS_NO_PATHCONV=1
|
|
|
|
rm -rf "$dir"
|
|
mkdir -p "$dir"
|
|
docker compose up -d --wait >/dev/null 2>&1
|
|
target=$(docker compose ps -q api | sed -n "${n}p")
|
|
[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; }
|
|
alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }')
|
|
meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \
|
|
"fault_at_s=$fault_at" "restore_at_s=$restore_at"
|
|
sh scripts/instances.sh >"$dir/instances-before.txt"
|
|
# A margin for the host clock, which may run ahead of the VM's.
|
|
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
|
|
|
|
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
|
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
|
generator=$!
|
|
# The generator seeds two users first; its timeline starts after that. The
|
|
# fault times below are absolute, as is started_at in timeline.json.
|
|
sleep "$fault_at"
|
|
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
docker "$kind" "$target" >/dev/null
|
|
exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target")
|
|
sleep $((restore_at - fault_at))
|
|
restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
docker compose start api >/dev/null 2>&1
|
|
started=$(docker inspect --format '{{.State.StartedAt}}' "$target")
|
|
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
|
|
|
|
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \
|
|
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json"
|
|
docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1
|
|
sh scripts/instances.sh >"$dir/instances-after.txt"
|
|
cat "$dir/loadgen.log" | tail -n 1
|
|
echo "results in $dir"
|