#!/bin/sh # Level 4: one API replica fails under a constant write load. The generator # sends messages at RATE for DURATION and retries uncertain sends with the # same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM, # graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started # again. Afterwards the generator reads the history back: every confirmed # message once. # # sh scripts/bench/failover.sh stop|kill # # Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), # OUT (default evidence/failover). Results in /-/: # timeline.json, seconds.csv (per-second outcomes, attributed by send time), # faults.json (when and what was stopped), compose.log (the balancer's # DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. # Runs on the current stand (`make up`) and leaves every replica running. # # Every time used for the analysis comes from the Docker VM's clock: the log # timestamps, the container's stop and start times from `docker inspect`, and # the generator's own times. The host clock of Windows runs apart from it by # a fraction of a second, a large share of a 1.5 s detection window; host # times are kept for reference only. With `stop`, the moment SIGTERM arrived # is the replica's own "shutdown started" line in compose.log. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh kind=${1:?usage: failover.sh stop|kill } n=${2:?usage: failover.sh stop|kill } case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac rate=${RATE:-50} duration=${DURATION:-60s} fault_at=${FAULT_AT:-15} restore_at=${RESTORE_AT:-35} dir=${OUT:-evidence/failover}/$kind-$n export MSYS_NO_PATHCONV=1 rm -rf "$dir" mkdir -p "$dir" docker compose up -d --wait >/dev/null 2>&1 target=$(docker compose ps -q api | sed -n "${n}p") [ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; } alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }') meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ "fault_at_s=$fault_at" "restore_at_s=$restore_at" sh scripts/instances.sh >"$dir/instances-before.txt" # A margin for the host clock, which may run ahead of the VM's. since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & generator=$! # The generator seeds two users first; its timeline starts after that. The # fault times below are absolute, as is started_at in timeline.json. sleep "$fault_at" fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker "$kind" "$target" >/dev/null exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target") sleep $((restore_at - fault_at)) restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker compose start api >/dev/null 2>&1 started=$(docker inspect --format '{{.State.StartedAt}}' "$target") wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \ "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json" docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1 sh scripts/instances.sh >"$dir/instances-after.txt" cat "$dir/loadgen.log" | tail -n 1 echo "results in $dir"