47 lines
2.1 KiB
Bash
47 lines
2.1 KiB
Bash
#!/usr/bin/env bash
|
|
set -eu
|
|
# Compose does not provide a native rolling-update primitive. This script
|
|
# performs a surge replacement: add one new healthy replica, then remove one old.
|
|
desired="${API_REPLICAS:-2}"
|
|
load_duration="${ROLLING_LOAD_DURATION:-90}"
|
|
load_script="${LOAD_SCRIPT:-tests/load.py}"
|
|
load_output="${ROLLING_LOAD_OUTPUT:-evidence/rolling-update-load.json}"
|
|
load_log="${ROLLING_LOAD_LOG:-evidence/rolling-update-load.log}"
|
|
if [ "$desired" -lt 2 ]; then echo 'API_REPLICAS must be >= 2' >&2; exit 2; fi
|
|
|
|
docker compose build api
|
|
python "$load_script" --base-url http://localhost:8080 --duration "$load_duration" --concurrency 8 --output "$load_output" >"$load_log" 2>&1 &
|
|
load_pid=$!
|
|
cleanup() { kill "$load_pid" 2>/dev/null || true; wait "$load_pid" 2>/dev/null || true; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# Capture the old replica ids. Each iteration adds a new container before removing an old one.
|
|
old_ids=$(docker compose ps -q api)
|
|
for old_id in $old_ids; do
|
|
before=$(docker compose ps -q api | sort)
|
|
docker compose up -d --no-deps --no-recreate --scale "api=$((desired + 1))" api
|
|
new_id=''
|
|
attempt=0
|
|
while [ "$attempt" -lt 60 ]; do
|
|
after=$(docker compose ps -q api | sort)
|
|
new_id=$(comm -13 <(printf '%s\n' "$before") <(printf '%s\n' "$after") | head -n 1 || true)
|
|
if [ -n "$new_id" ]; then
|
|
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$new_id")
|
|
if [ "$health" = healthy ]; then break; fi
|
|
fi
|
|
attempt=$((attempt + 1)); sleep 2
|
|
done
|
|
if [ -z "$new_id" ] || [ "${health:-}" != healthy ]; then
|
|
echo 'New replica did not become healthy; leaving old replicas running' >&2
|
|
exit 1
|
|
fi
|
|
docker exec "$old_id" sh -c 'touch /tmp/api-draining'
|
|
sleep 10
|
|
docker stop --timeout 20 "$old_id" >/dev/null
|
|
docker rm "$old_id" >/dev/null
|
|
docker compose up -d --no-deps --no-recreate --scale "api=$desired" api
|
|
python scripts/wait-ready.py http://localhost:8080/health/ready 30
|
|
done
|
|
wait "$load_pid" || { echo 'Load test failed; inspect evidence/rolling-update-load.log' >&2; exit 1; }
|
|
trap - EXIT INT TERM
|