Files
mtusi2026lab4/scripts/load/failover-under-load.sh
T
2026-11-04 22:57:50 +03:00

56 lines
2.5 KiB
Bash
Executable File

#!/bin/sh
# One replica fault under constant load through the balancer, then its restoration.
# MODE=stop docker stop (SIGTERM: drain, graceful shutdown) MODE=kill docker kill (SIGKILL: crash)
# TARGET=api-1|api-2 RATE=150 req/s DURATION=45 s fault at 10 s, restore at 25 s
# Output: evidence/load/failover-<mode>-<target>/{k6.txt,events.json,haproxy.log,per-second.csv,summary.json}
set -eu
cd "$(dirname "$0")/../.."
MODE=${MODE:-stop}
TARGET=${TARGET:-api-1}
RATE=${RATE:-150}
DURATION=${DURATION:-45}
FAULT_AT=${FAULT_AT:-10}
RESTORE_AT=${RESTORE_AT:-25}
out=evidence/load/failover-$MODE-$TARGET
rm -rf "$out" && mkdir -p "$out"
now() { python3 -c 'import time; print(int(time.time() * 1000))'; }
# Sleeps until <seconds> after the load started, so the schedule does not drift with slow docker commands.
sleep_until() { python3 -c "import time; time.sleep(max(0, $start / 1000 + $1 - time.time()))"; }
container=mtusi-messenger-lab4-$TARGET
target_instance=$(docker inspect -f '{{.Config.Hostname}}' "$container")
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
K6_CSV_TIME_FORMAT=unix_milli k6 run --quiet --out csv="$out/requests.csv" \
-e RATE="$RATE" -e DURATION="${DURATION}s" scripts/load/mixed.js > "$out/k6.txt" 2>&1 &
k6_pid=$!
# Load starts after k6 setup (users, sessions, chat): wait for the first request of the scenario.
until grep -q ',load,' "$out/requests.csv" 2>/dev/null; do sleep 0.2; done
start=$(now)
sleep_until "$FAULT_AT"
fault=$(now)
if [ "$MODE" = kill ]; then docker kill "$container" >/dev/null; else docker stop --time 9 "$container" >/dev/null; fi
fault_done=$(now)
sleep_until "$RESTORE_AT"
restore_begin=$(now)
docker start "$container" >/dev/null
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" = healthy ]; do sleep 0.2; done
restore_healthy=$(now)
restored_instance=$(docker inspect -f '{{.Config.Hostname}}' "$container")
wait "$k6_pid" || true
docker compose logs --timestamps --no-log-prefix --since "$since" lb 2>/dev/null | grep -E 'Server api/|SH--|5[0-9][0-9] ' > "$out/haproxy.log" || true
cat > "$out/events.json" <<JSON
{"mode": "$MODE", "target": "$TARGET", "target_instance": "$target_instance", "restored_instance": "$restored_instance",
"load_start": $start, "fault": $fault, "fault_done": $fault_done, "restore_begin": $restore_begin, "restore_healthy": $restore_healthy}
JSON
python3 scripts/load/summarize.py "$out" > /dev/null
# Raw per-request data stays out of Git (evidence keeps aggregates only); KEEP_RAW=1 keeps it for analysis.
[ "${KEEP_RAW:-0}" = 1 ] || rm -f "$out/requests.csv"
cat "$out/summary.json"