forked from mtusi2026/mtusi2026lab3
239 lines
6.2 KiB
Bash
Executable File
239 lines
6.2 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
cd "$(dirname "$0")/.."
|
|
|
|
if [ -f .env ]; then
|
|
compose() { docker compose --env-file .env "$@"; }
|
|
else
|
|
: "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}"
|
|
compose() { docker compose "$@"; }
|
|
fi
|
|
|
|
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
|
base_url=${base_url%/}
|
|
|
|
container_ip() {
|
|
docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
|
|
}
|
|
|
|
haproxy_cli() {
|
|
docker exec "$lb_container" sh -c \
|
|
'printf "%s\n" "$1" | socat - TCP4:127.0.0.1:9999' sh "$1"
|
|
}
|
|
|
|
slot_for_ip() {
|
|
haproxy_cli 'show servers state api_replicas' |
|
|
awk -v address="$1" '$2 == "api_replicas" && $5 == address { print $4; exit }'
|
|
}
|
|
|
|
ip_for_slot() {
|
|
haproxy_cli 'show servers state api_replicas' |
|
|
awk -v name="$1" '$2 == "api_replicas" && $4 == name { print $5; exit }'
|
|
}
|
|
|
|
set_server_state() {
|
|
response=$(haproxy_cli "set server api_replicas/$1 state $2")
|
|
if [ -n "$response" ]; then
|
|
printf 'HAProxy rejected state change for %s: %s\n' "$1" "$response" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
wait_drained() {
|
|
attempts=0
|
|
while [ "$attempts" -lt 30 ]; do
|
|
active=$(haproxy_cli 'show stat' |
|
|
awk -F, -v name="$1" '$1 == "api_replicas" && $2 == name { print $5; exit }')
|
|
if [ "$active" = 0 ]; then
|
|
return 0
|
|
fi
|
|
attempts=$((attempts + 1))
|
|
sleep 1
|
|
done
|
|
printf 'HAProxy backend %s did not drain\n' "$1" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_lb_slot_for_ip() {
|
|
attempts=0
|
|
while [ "$attempts" -lt 30 ]; do
|
|
found=$(slot_for_ip "$1")
|
|
if [ -n "$found" ]; then
|
|
printf '%s\n' "$found"
|
|
return 0
|
|
fi
|
|
attempts=$((attempts + 1))
|
|
sleep 1
|
|
done
|
|
printf 'HAProxy did not resolve new API address %s\n' "$1" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_old_address_released() {
|
|
attempts=0
|
|
while [ "$attempts" -lt 30 ]; do
|
|
if [ "$(ip_for_slot "$1")" != "$2" ]; then
|
|
return 0
|
|
fi
|
|
attempts=$((attempts + 1))
|
|
sleep 1
|
|
done
|
|
printf 'HAProxy kept retired API address %s on %s\n' "$2" "$1" >&2
|
|
return 1
|
|
}
|
|
|
|
container_except() {
|
|
excluded=$1
|
|
for candidate in $(compose ps -aq api); do
|
|
if [ "$candidate" != "$excluded" ]; then
|
|
printf '%s\n' "$candidate"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
is_ready() {
|
|
docker exec "$1" wget -q -O /dev/null http://127.0.0.1:8080/health/ready >/dev/null 2>&1
|
|
}
|
|
|
|
ready_count() {
|
|
count=0
|
|
for candidate in $(compose ps -q api); do
|
|
if is_ready "$candidate"; then
|
|
count=$((count + 1))
|
|
fi
|
|
done
|
|
printf '%s\n' "$count"
|
|
}
|
|
|
|
wait_ready() {
|
|
candidate=$1
|
|
attempts=0
|
|
while [ "$attempts" -lt 30 ]; do
|
|
if is_ready "$candidate"; then
|
|
return 0
|
|
fi
|
|
attempts=$((attempts + 1))
|
|
sleep 1
|
|
done
|
|
printf 'API container did not become ready: %s\n' "$candidate" >&2
|
|
return 1
|
|
}
|
|
|
|
wait_for_lb() {
|
|
candidate=$1
|
|
expected=$(docker inspect --format '{{.Config.Hostname}}' "$candidate")
|
|
attempts=0
|
|
while [ "$attempts" -lt 30 ]; do
|
|
observed=$(curl -sS --max-time 2 -D - -o /dev/null "$base_url/health/ready" 2>/dev/null |
|
|
awk 'tolower($1) == "x-instance-id:" { gsub("\r", "", $2); print $2; exit }')
|
|
if [ "$observed" = "$expected" ]; then
|
|
return 0
|
|
fi
|
|
attempts=$((attempts + 1))
|
|
sleep 1
|
|
done
|
|
printf 'HAProxy did not route to new API container: %s\n' "$candidate" >&2
|
|
return 1
|
|
}
|
|
|
|
set -- $(compose ps -q api)
|
|
if [ "$#" -ne 2 ] || ! is_ready "$1" || ! is_ready "$2"; then
|
|
printf '%s\n' 'Two ready API replicas are required before replacement' >&2
|
|
exit 1
|
|
fi
|
|
first=$1
|
|
second=$2
|
|
lb_container=$(compose ps -q lb)
|
|
if [ -z "$lb_container" ] ||
|
|
[ -z "$(slot_for_ip "$(container_ip "$first")")" ] ||
|
|
[ -z "$(slot_for_ip "$(container_ip "$second")")" ]; then
|
|
printf '%s\n' 'HAProxy Runtime API did not expose both ready API backends' >&2
|
|
exit 1
|
|
fi
|
|
|
|
min_ready=2
|
|
completed=0
|
|
drained_slot=
|
|
restore_on_failure() {
|
|
if [ "$completed" -ne 1 ]; then
|
|
compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2 || true
|
|
if [ -n "$drained_slot" ]; then
|
|
set_server_state "$drained_slot" ready >&2 || true
|
|
fi
|
|
fi
|
|
}
|
|
trap restore_on_failure 0
|
|
|
|
for step in 1 2; do
|
|
if [ "$step" -eq 1 ]; then
|
|
old=$first
|
|
survivor=$second
|
|
else
|
|
old=$second
|
|
survivor=$first
|
|
fi
|
|
|
|
if [ "$(container_except "$survivor")" != "$old" ] || ! is_ready "$survivor"; then
|
|
printf 'API replica %s changed or survivor is not ready\n' "$step" >&2
|
|
exit 1
|
|
fi
|
|
|
|
old_ip=$(container_ip "$old")
|
|
old_slot=$(slot_for_ip "$old_ip")
|
|
if [ -z "$old_slot" ]; then
|
|
printf 'HAProxy backend for %s not found\n' "$old" >&2
|
|
exit 1
|
|
fi
|
|
set_server_state "$old_slot" drain
|
|
drained_slot=$old_slot
|
|
wait_drained "$old_slot"
|
|
sleep 1
|
|
|
|
docker stop --timeout 18 "$old" >/dev/null
|
|
current_ready=$(ready_count)
|
|
if [ "$current_ready" -lt 1 ]; then
|
|
printf '%s\n' 'No ready API replica remained during replacement' >&2
|
|
exit 1
|
|
fi
|
|
if [ "$current_ready" -lt "$min_ready" ]; then
|
|
min_ready=$current_ready
|
|
fi
|
|
|
|
docker rm "$old" >/dev/null
|
|
compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2
|
|
|
|
new=$(container_except "$survivor")
|
|
if [ "$new" = "$old" ] || ! is_ready "$survivor"; then
|
|
printf 'Compose unexpectedly replaced another API replica at step %s\n' "$step" >&2
|
|
exit 1
|
|
fi
|
|
wait_ready "$new"
|
|
new_ip=$(container_ip "$new")
|
|
new_slot=$(wait_lb_slot_for_ip "$new_ip")
|
|
if [ "$new_ip" != "$old_ip" ]; then
|
|
wait_old_address_released "$old_slot" "$old_ip"
|
|
fi
|
|
set_server_state "$new_slot" ready
|
|
if [ "$new_slot" != "$old_slot" ]; then
|
|
set_server_state "$old_slot" ready
|
|
fi
|
|
drained_slot=
|
|
wait_for_lb "$new"
|
|
if [ "$step" -eq 1 ]; then
|
|
first=$new
|
|
else
|
|
second=$new
|
|
fi
|
|
done
|
|
|
|
if [ "$(compose ps -q api | wc -l | tr -d ' ')" -ne 2 ]; then
|
|
printf '%s\n' 'Expected exactly two running API replicas after replacement' >&2
|
|
exit 1
|
|
fi
|
|
|
|
completed=1
|
|
printf '{"replaced_processes":2,"min_ready_during_update":%s}\n' "$min_ready"
|