#!/bin/sh # Stop each original pod explicitly and preserve request and Kubernetes evidence. set -eu cd "$(dirname "$0")/.." : "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" : "${EVIDENCE_DIR:=evidence/review-20261029}" mkdir -p "$EVIDENCE_DIR" if [ "${RESUME_FROM:-1}" = 1 ]; then kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l app=api -o json | jq ' .items |= map(select(.metadata.deletionTimestamp == null)) ' > "$EVIDENCE_DIR/initial-api.json" test "$(jq '.items|length' "$EVIDENCE_DIR/initial-api.json")" -eq 2 jq -r '.items[].metadata.name' "$EVIDENCE_DIR/initial-api.json" > "$EVIDENCE_DIR/targets.txt" fi index=0 while read -r target; do index=$((index+1)) [ "$index" -ge "${RESUME_FROM:-1}" ] || continue run="failover-$index-$(date -u +%Y%m%d%H%M%S)" TARGET_INSTANCE_ID="$target" RUN="$run" LOAD_SCRIPT=failover.js EVIDENCE_DIR="$EVIDENCE_DIR" scripts/run-load.sh > "$EVIDENCE_DIR/$run-run.txt" 2>&1 & runner=$! ready=false for attempt in $(seq 1 90); do if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/load-$run" 2>/dev/null | rg -q READY_TO_STOP; then ready=true; break; fi kill -0 "$runner" 2>/dev/null || break sleep 1 done if ! $ready; then wait "$runner" || true; echo 'Failover setup did not become ready' >&2; exit 1; fi ruby -e 'puts Time.now.to_f' > "$EVIDENCE_DIR/$run-stop-epoch.txt" kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$target" -o json > "$EVIDENCE_DIR/$run-target.json" kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$target" --wait=false > "$EVIDENCE_DIR/$run-delete.txt" wait "$runner" ruby scripts/summarize-failover.rb "$EVIDENCE_DIR" "$run" kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s > "$EVIDENCE_DIR/$run-restored.txt" done < "$EVIDENCE_DIR/targets.txt"