Files
mtusi2026lab5/scripts/bench/failover.sh
T

57 lines
2.7 KiB
Bash
Executable File

#!/bin/sh
# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and
# started again at RESTORE_AT, under a constant write load that retries
# uncertain sends with the same key; then every confirmed message must be in
# the history once.
#
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
#
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT
# (evidence/failover). Times come from the Docker VM clock; the Windows clock
# differs by a fraction of a second and is kept for reference only.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
kind=${1:?usage: failover.sh stop|kill <replica number>}
n=${2:?usage: failover.sh stop|kill <replica number>}
case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac
rate=${RATE:-50}
duration=${DURATION:-60s}
fault_at=${FAULT_AT:-15}
restore_at=${RESTORE_AT:-35}
dir=${OUT:-evidence/failover}/$kind-$n
export MSYS_NO_PATHCONV=1
rm -rf "$dir"
mkdir -p "$dir"
docker compose up -d --wait >/dev/null 2>&1
target=$(docker compose ps -q api | sed -n "${n}p")
[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; }
alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }')
meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \
"fault_at_s=$fault_at" "restore_at_s=$restore_at"
sh scripts/instances.sh >"$dir/instances-before.txt"
# A margin for the host clock, which may run ahead of the VM's.
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
generator=$!
sleep "$fault_at"
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker "$kind" "$target" >/dev/null
exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target")
sleep $((restore_at - fault_at))
restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker compose start api >/dev/null 2>&1
started=$(docker inspect --format '{{.State.StartedAt}}' "$target")
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json"
docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1
sh scripts/instances.sh >"$dir/instances-after.txt"
cat "$dir/loadgen.log" | tail -n 1
echo "results in $dir"