forked from mtusi2026/mtusi2026lab5
56 lines
2.5 KiB
Bash
Executable File
56 lines
2.5 KiB
Bash
Executable File
#!/bin/sh
|
|
# One replica fault under constant load through the balancer, then its restoration.
|
|
# MODE=stop docker stop (SIGTERM: drain, graceful shutdown) MODE=kill docker kill (SIGKILL: crash)
|
|
# TARGET=api-1|api-2 RATE=150 req/s DURATION=45 s fault at 10 s, restore at 25 s
|
|
# Output: evidence/load/failover-<mode>-<target>/{k6.txt,events.json,haproxy.log,per-second.csv,summary.json}
|
|
set -eu
|
|
cd "$(dirname "$0")/../.."
|
|
|
|
MODE=${MODE:-stop}
|
|
TARGET=${TARGET:-api-1}
|
|
RATE=${RATE:-150}
|
|
DURATION=${DURATION:-45}
|
|
FAULT_AT=${FAULT_AT:-10}
|
|
RESTORE_AT=${RESTORE_AT:-25}
|
|
|
|
out=evidence/load/failover-$MODE-$TARGET
|
|
rm -rf "$out" && mkdir -p "$out"
|
|
now() { python3 -c 'import time; print(int(time.time() * 1000))'; }
|
|
# Sleeps until <seconds> after the load started, so the schedule does not drift with slow docker commands.
|
|
sleep_until() { python3 -c "import time; time.sleep(max(0, $start / 1000 + $1 - time.time()))"; }
|
|
container=mtusi-messenger-lab5-$TARGET
|
|
target_instance=$(docker inspect -f '{{.Config.Hostname}}' "$container")
|
|
|
|
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
K6_CSV_TIME_FORMAT=unix_milli k6 run --quiet --out csv="$out/requests.csv" \
|
|
-e RATE="$RATE" -e DURATION="${DURATION}s" scripts/load/mixed.js > "$out/k6.txt" 2>&1 &
|
|
k6_pid=$!
|
|
|
|
# Load starts after k6 setup (users, sessions, chat): wait for the first request of the scenario.
|
|
until grep -q ',load,' "$out/requests.csv" 2>/dev/null; do sleep 0.2; done
|
|
start=$(now)
|
|
|
|
sleep_until "$FAULT_AT"
|
|
fault=$(now)
|
|
if [ "$MODE" = kill ]; then docker kill "$container" >/dev/null; else docker stop --time 9 "$container" >/dev/null; fi
|
|
fault_done=$(now)
|
|
|
|
sleep_until "$RESTORE_AT"
|
|
restore_begin=$(now)
|
|
docker start "$container" >/dev/null
|
|
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" = healthy ]; do sleep 0.2; done
|
|
restore_healthy=$(now)
|
|
restored_instance=$(docker inspect -f '{{.Config.Hostname}}' "$container")
|
|
|
|
wait "$k6_pid" || true
|
|
docker compose logs --timestamps --no-log-prefix --since "$since" lb 2>/dev/null | grep -E 'Server api/|SH--|5[0-9][0-9] ' > "$out/haproxy.log" || true
|
|
|
|
cat > "$out/events.json" <<JSON
|
|
{"mode": "$MODE", "target": "$TARGET", "target_instance": "$target_instance", "restored_instance": "$restored_instance",
|
|
"load_start": $start, "fault": $fault, "fault_done": $fault_done, "restore_begin": $restore_begin, "restore_healthy": $restore_healthy}
|
|
JSON
|
|
python3 scripts/load/summarize.py "$out" > /dev/null
|
|
# Raw per-request data stays out of Git (evidence keeps aggregates only); KEEP_RAW=1 keeps it for analysis.
|
|
[ "${KEEP_RAW:-0}" = 1 ] || rm -f "$out/requests.csv"
|
|
cat "$out/summary.json"
|