Files
mtusi2026lab3/scripts/bench/failover.sh
T

71 lines
3.6 KiB
Bash
Executable File

#!/bin/sh
# Level 4: one API replica fails under a constant write load. The generator
# sends messages at RATE for DURATION and retries uncertain sends with the
# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM,
# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started
# again. Afterwards the generator reads the history back: every confirmed
# message once.
#
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
#
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35),
# OUT (default evidence/failover). Results in <OUT>/<kind>-<n>/:
# timeline.json, seconds.csv (per-second outcomes, attributed by send time),
# faults.json (when and what was stopped), compose.log (the balancer's
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
# Runs on the current stand (`make up`) and leaves every replica running.
#
# Every time used for the analysis comes from the Docker VM's clock: the log
# timestamps, the container's stop and start times from `docker inspect`, and
# the generator's own times. The host clock of Windows runs apart from it by
# a fraction of a second, a large share of a 1.5 s detection window; host
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
# is the replica's own "shutdown started" line in compose.log.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
kind=${1:?usage: failover.sh stop|kill <replica number>}
n=${2:?usage: failover.sh stop|kill <replica number>}
case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac
rate=${RATE:-50}
duration=${DURATION:-60s}
fault_at=${FAULT_AT:-15}
restore_at=${RESTORE_AT:-35}
dir=${OUT:-evidence/failover}/$kind-$n
export MSYS_NO_PATHCONV=1
rm -rf "$dir"
mkdir -p "$dir"
docker compose up -d --wait >/dev/null 2>&1
target=$(docker compose ps -q api | sed -n "${n}p")
[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; }
alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }')
meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \
"fault_at_s=$fault_at" "restore_at_s=$restore_at"
sh scripts/instances.sh >"$dir/instances-before.txt"
# A margin for the host clock, which may run ahead of the VM's.
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
generator=$!
# The generator seeds two users first; its timeline starts after that. The
# fault times below are absolute, as is started_at in timeline.json.
sleep "$fault_at"
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker "$kind" "$target" >/dev/null
exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target")
sleep $((restore_at - fault_at))
restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker compose start api >/dev/null 2>&1
started=$(docker inspect --format '{{.State.StartedAt}}' "$target")
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json"
docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1
sh scripts/instances.sh >"$dir/instances-after.txt"
cat "$dir/loadgen.log" | tail -n 1
echo "results in $dir"