#!/bin/sh set -eu cd "$(dirname "$0")/.." replicas() { docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}") } url=${1:-http://localhost:8080} shift=0 while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done tmp=evidence/failover.tmp since=$(date -u +%FT%TZ) { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt" replicas echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh" make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url" echo "== after: api replicas" replicas } > "$tmp" 2>&1 stopped=$(replicas | awk '$2 == "exited" {print $1}') { echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit" docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16 echo "\$ scripts/restore.sh" scripts/restore.sh echo "== after restore" replicas } >> "$tmp" 2>&1 mv "$tmp" "evidence/failover-$stopped.txt" cat "evidence/failover-$stopped.txt"