forked from mtusi2026/mtusi2026lab3
34 lines
1.9 KiB
Bash
Executable File
34 lines
1.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# Stop each original pod explicitly and preserve request and Kubernetes evidence.
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
|
: "${EVIDENCE_DIR:=evidence/review-20261029}"
|
|
mkdir -p "$EVIDENCE_DIR"
|
|
if [ "${RESUME_FROM:-1}" = 1 ]; then
|
|
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l app=api -o json | jq ' .items |= map(select(.metadata.deletionTimestamp == null)) ' > "$EVIDENCE_DIR/initial-api.json"
|
|
test "$(jq '.items|length' "$EVIDENCE_DIR/initial-api.json")" -eq 2
|
|
jq -r '.items[].metadata.name' "$EVIDENCE_DIR/initial-api.json" > "$EVIDENCE_DIR/targets.txt"
|
|
fi
|
|
index=0
|
|
while read -r target; do
|
|
index=$((index+1))
|
|
[ "$index" -ge "${RESUME_FROM:-1}" ] || continue
|
|
run="failover-$index-$(date -u +%Y%m%d%H%M%S)"
|
|
TARGET_INSTANCE_ID="$target" RUN="$run" LOAD_SCRIPT=failover.js EVIDENCE_DIR="$EVIDENCE_DIR" scripts/run-load.sh > "$EVIDENCE_DIR/$run-run.txt" 2>&1 &
|
|
runner=$!
|
|
ready=false
|
|
for attempt in $(seq 1 90); do
|
|
if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/load-$run" 2>/dev/null | rg -q READY_TO_STOP; then ready=true; break; fi
|
|
kill -0 "$runner" 2>/dev/null || break
|
|
sleep 1
|
|
done
|
|
if ! $ready; then wait "$runner" || true; echo 'Failover setup did not become ready' >&2; exit 1; fi
|
|
ruby -e 'puts Time.now.to_f' > "$EVIDENCE_DIR/$run-stop-epoch.txt"
|
|
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$target" -o json > "$EVIDENCE_DIR/$run-target.json"
|
|
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$target" --wait=false > "$EVIDENCE_DIR/$run-delete.txt"
|
|
wait "$runner"
|
|
ruby scripts/summarize-failover.rb "$EVIDENCE_DIR" "$run"
|
|
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s > "$EVIDENCE_DIR/$run-restored.txt"
|
|
done < "$EVIDENCE_DIR/targets.txt"
|