Files
mtusi2026lab3/scripts/rolling-update.sh
T

239 lines
6.2 KiB
Bash
Executable File

#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ -f .env ]; then
compose() { docker compose --env-file .env "$@"; }
else
: "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}"
compose() { docker compose "$@"; }
fi
base_url=${BASE_URL:-http://127.0.0.1:8080}
base_url=${base_url%/}
container_ip() {
docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
}
haproxy_cli() {
docker exec "$lb_container" sh -c \
'printf "%s\n" "$1" | socat - TCP4:127.0.0.1:9999' sh "$1"
}
slot_for_ip() {
haproxy_cli 'show servers state api_replicas' |
awk -v address="$1" '$2 == "api_replicas" && $5 == address { print $4; exit }'
}
ip_for_slot() {
haproxy_cli 'show servers state api_replicas' |
awk -v name="$1" '$2 == "api_replicas" && $4 == name { print $5; exit }'
}
set_server_state() {
response=$(haproxy_cli "set server api_replicas/$1 state $2")
if [ -n "$response" ]; then
printf 'HAProxy rejected state change for %s: %s\n' "$1" "$response" >&2
return 1
fi
}
wait_drained() {
attempts=0
while [ "$attempts" -lt 30 ]; do
active=$(haproxy_cli 'show stat' |
awk -F, -v name="$1" '$1 == "api_replicas" && $2 == name { print $5; exit }')
if [ "$active" = 0 ]; then
return 0
fi
attempts=$((attempts + 1))
sleep 1
done
printf 'HAProxy backend %s did not drain\n' "$1" >&2
return 1
}
wait_lb_slot_for_ip() {
attempts=0
while [ "$attempts" -lt 30 ]; do
found=$(slot_for_ip "$1")
if [ -n "$found" ]; then
printf '%s\n' "$found"
return 0
fi
attempts=$((attempts + 1))
sleep 1
done
printf 'HAProxy did not resolve new API address %s\n' "$1" >&2
return 1
}
wait_old_address_released() {
attempts=0
while [ "$attempts" -lt 30 ]; do
if [ "$(ip_for_slot "$1")" != "$2" ]; then
return 0
fi
attempts=$((attempts + 1))
sleep 1
done
printf 'HAProxy kept retired API address %s on %s\n' "$2" "$1" >&2
return 1
}
container_except() {
excluded=$1
for candidate in $(compose ps -aq api); do
if [ "$candidate" != "$excluded" ]; then
printf '%s\n' "$candidate"
return 0
fi
done
return 1
}
is_ready() {
docker exec "$1" wget -q -O /dev/null http://127.0.0.1:8080/health/ready >/dev/null 2>&1
}
ready_count() {
count=0
for candidate in $(compose ps -q api); do
if is_ready "$candidate"; then
count=$((count + 1))
fi
done
printf '%s\n' "$count"
}
wait_ready() {
candidate=$1
attempts=0
while [ "$attempts" -lt 30 ]; do
if is_ready "$candidate"; then
return 0
fi
attempts=$((attempts + 1))
sleep 1
done
printf 'API container did not become ready: %s\n' "$candidate" >&2
return 1
}
wait_for_lb() {
candidate=$1
expected=$(docker inspect --format '{{.Config.Hostname}}' "$candidate")
attempts=0
while [ "$attempts" -lt 30 ]; do
observed=$(curl -sS --max-time 2 -D - -o /dev/null "$base_url/health/ready" 2>/dev/null |
awk 'tolower($1) == "x-instance-id:" { gsub("\r", "", $2); print $2; exit }')
if [ "$observed" = "$expected" ]; then
return 0
fi
attempts=$((attempts + 1))
sleep 1
done
printf 'HAProxy did not route to new API container: %s\n' "$candidate" >&2
return 1
}
set -- $(compose ps -q api)
if [ "$#" -ne 2 ] || ! is_ready "$1" || ! is_ready "$2"; then
printf '%s\n' 'Two ready API replicas are required before replacement' >&2
exit 1
fi
first=$1
second=$2
lb_container=$(compose ps -q lb)
if [ -z "$lb_container" ] ||
[ -z "$(slot_for_ip "$(container_ip "$first")")" ] ||
[ -z "$(slot_for_ip "$(container_ip "$second")")" ]; then
printf '%s\n' 'HAProxy Runtime API did not expose both ready API backends' >&2
exit 1
fi
min_ready=2
completed=0
drained_slot=
restore_on_failure() {
if [ "$completed" -ne 1 ]; then
compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2 || true
if [ -n "$drained_slot" ]; then
set_server_state "$drained_slot" ready >&2 || true
fi
fi
}
trap restore_on_failure 0
for step in 1 2; do
if [ "$step" -eq 1 ]; then
old=$first
survivor=$second
else
old=$second
survivor=$first
fi
if [ "$(container_except "$survivor")" != "$old" ] || ! is_ready "$survivor"; then
printf 'API replica %s changed or survivor is not ready\n' "$step" >&2
exit 1
fi
old_ip=$(container_ip "$old")
old_slot=$(slot_for_ip "$old_ip")
if [ -z "$old_slot" ]; then
printf 'HAProxy backend for %s not found\n' "$old" >&2
exit 1
fi
set_server_state "$old_slot" drain
drained_slot=$old_slot
wait_drained "$old_slot"
sleep 1
docker stop --timeout 18 "$old" >/dev/null
current_ready=$(ready_count)
if [ "$current_ready" -lt 1 ]; then
printf '%s\n' 'No ready API replica remained during replacement' >&2
exit 1
fi
if [ "$current_ready" -lt "$min_ready" ]; then
min_ready=$current_ready
fi
docker rm "$old" >/dev/null
compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2
new=$(container_except "$survivor")
if [ "$new" = "$old" ] || ! is_ready "$survivor"; then
printf 'Compose unexpectedly replaced another API replica at step %s\n' "$step" >&2
exit 1
fi
wait_ready "$new"
new_ip=$(container_ip "$new")
new_slot=$(wait_lb_slot_for_ip "$new_ip")
if [ "$new_ip" != "$old_ip" ]; then
wait_old_address_released "$old_slot" "$old_ip"
fi
set_server_state "$new_slot" ready
if [ "$new_slot" != "$old_slot" ]; then
set_server_state "$old_slot" ready
fi
drained_slot=
wait_for_lb "$new"
if [ "$step" -eq 1 ]; then
first=$new
else
second=$new
fi
done
if [ "$(compose ps -q api | wc -l | tr -d ' ')" -ne 2 ]; then
printf '%s\n' 'Expected exactly two running API replicas after replacement' >&2
exit 1
fi
completed=1
printf '{"replaced_processes":2,"min_ready_during_update":%s}\n' "$min_ready"