Implement lab 3 level 4 failover and idempotency
This commit is contained in:
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ -f .env ]; then
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
else
|
||||
: "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}"
|
||||
compose() { docker compose "$@"; }
|
||||
fi
|
||||
|
||||
# Keep named volumes and the one-shot migrator untouched.
|
||||
compose restart --no-deps --timeout 20 postgres redis
|
||||
compose restart --no-deps --timeout 20 api
|
||||
compose restart --no-deps --timeout 20 lb
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ -f .env ]; then
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
else
|
||||
: "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}"
|
||||
compose() { docker compose "$@"; }
|
||||
fi
|
||||
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
base_url=${base_url%/}
|
||||
|
||||
container_ip() {
|
||||
docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
|
||||
}
|
||||
|
||||
haproxy_cli() {
|
||||
docker exec "$lb_container" sh -c \
|
||||
'printf "%s\n" "$1" | socat - TCP4:127.0.0.1:9999' sh "$1"
|
||||
}
|
||||
|
||||
slot_for_ip() {
|
||||
haproxy_cli 'show servers state api_replicas' |
|
||||
awk -v address="$1" '$2 == "api_replicas" && $5 == address { print $4; exit }'
|
||||
}
|
||||
|
||||
ip_for_slot() {
|
||||
haproxy_cli 'show servers state api_replicas' |
|
||||
awk -v name="$1" '$2 == "api_replicas" && $4 == name { print $5; exit }'
|
||||
}
|
||||
|
||||
set_server_state() {
|
||||
response=$(haproxy_cli "set server api_replicas/$1 state $2")
|
||||
if [ -n "$response" ]; then
|
||||
printf 'HAProxy rejected state change for %s: %s\n' "$1" "$response" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
wait_drained() {
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
active=$(haproxy_cli 'show stat' |
|
||||
awk -F, -v name="$1" '$1 == "api_replicas" && $2 == name { print $5; exit }')
|
||||
if [ "$active" = 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
printf 'HAProxy backend %s did not drain\n' "$1" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_lb_slot_for_ip() {
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
found=$(slot_for_ip "$1")
|
||||
if [ -n "$found" ]; then
|
||||
printf '%s\n' "$found"
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
printf 'HAProxy did not resolve new API address %s\n' "$1" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_old_address_released() {
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
if [ "$(ip_for_slot "$1")" != "$2" ]; then
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
printf 'HAProxy kept retired API address %s on %s\n' "$2" "$1" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
container_except() {
|
||||
excluded=$1
|
||||
for candidate in $(compose ps -aq api); do
|
||||
if [ "$candidate" != "$excluded" ]; then
|
||||
printf '%s\n' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_ready() {
|
||||
docker exec "$1" wget -q -O /dev/null http://127.0.0.1:8080/health/ready >/dev/null 2>&1
|
||||
}
|
||||
|
||||
ready_count() {
|
||||
count=0
|
||||
for candidate in $(compose ps -q api); do
|
||||
if is_ready "$candidate"; then
|
||||
count=$((count + 1))
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "$count"
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
candidate=$1
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
if is_ready "$candidate"; then
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
printf 'API container did not become ready: %s\n' "$candidate" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_lb() {
|
||||
candidate=$1
|
||||
expected=$(docker inspect --format '{{.Config.Hostname}}' "$candidate")
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
observed=$(curl -sS --max-time 2 -D - -o /dev/null "$base_url/health/ready" 2>/dev/null |
|
||||
awk 'tolower($1) == "x-instance-id:" { gsub("\r", "", $2); print $2; exit }')
|
||||
if [ "$observed" = "$expected" ]; then
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
printf 'HAProxy did not route to new API container: %s\n' "$candidate" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
set -- $(compose ps -q api)
|
||||
if [ "$#" -ne 2 ] || ! is_ready "$1" || ! is_ready "$2"; then
|
||||
printf '%s\n' 'Two ready API replicas are required before replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
first=$1
|
||||
second=$2
|
||||
lb_container=$(compose ps -q lb)
|
||||
if [ -z "$lb_container" ] ||
|
||||
[ -z "$(slot_for_ip "$(container_ip "$first")")" ] ||
|
||||
[ -z "$(slot_for_ip "$(container_ip "$second")")" ]; then
|
||||
printf '%s\n' 'HAProxy Runtime API did not expose both ready API backends' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
min_ready=2
|
||||
completed=0
|
||||
drained_slot=
|
||||
restore_on_failure() {
|
||||
if [ "$completed" -ne 1 ]; then
|
||||
compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2 || true
|
||||
if [ -n "$drained_slot" ]; then
|
||||
set_server_state "$drained_slot" ready >&2 || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
trap restore_on_failure 0
|
||||
|
||||
for step in 1 2; do
|
||||
if [ "$step" -eq 1 ]; then
|
||||
old=$first
|
||||
survivor=$second
|
||||
else
|
||||
old=$second
|
||||
survivor=$first
|
||||
fi
|
||||
|
||||
if [ "$(container_except "$survivor")" != "$old" ] || ! is_ready "$survivor"; then
|
||||
printf 'API replica %s changed or survivor is not ready\n' "$step" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
old_ip=$(container_ip "$old")
|
||||
old_slot=$(slot_for_ip "$old_ip")
|
||||
if [ -z "$old_slot" ]; then
|
||||
printf 'HAProxy backend for %s not found\n' "$old" >&2
|
||||
exit 1
|
||||
fi
|
||||
set_server_state "$old_slot" drain
|
||||
drained_slot=$old_slot
|
||||
wait_drained "$old_slot"
|
||||
sleep 1
|
||||
|
||||
docker stop --timeout 18 "$old" >/dev/null
|
||||
current_ready=$(ready_count)
|
||||
if [ "$current_ready" -lt 1 ]; then
|
||||
printf '%s\n' 'No ready API replica remained during replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$current_ready" -lt "$min_ready" ]; then
|
||||
min_ready=$current_ready
|
||||
fi
|
||||
|
||||
docker rm "$old" >/dev/null
|
||||
compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2
|
||||
|
||||
new=$(container_except "$survivor")
|
||||
if [ "$new" = "$old" ] || ! is_ready "$survivor"; then
|
||||
printf 'Compose unexpectedly replaced another API replica at step %s\n' "$step" >&2
|
||||
exit 1
|
||||
fi
|
||||
wait_ready "$new"
|
||||
new_ip=$(container_ip "$new")
|
||||
new_slot=$(wait_lb_slot_for_ip "$new_ip")
|
||||
if [ "$new_ip" != "$old_ip" ]; then
|
||||
wait_old_address_released "$old_slot" "$old_ip"
|
||||
fi
|
||||
set_server_state "$new_slot" ready
|
||||
if [ "$new_slot" != "$old_slot" ]; then
|
||||
set_server_state "$old_slot" ready
|
||||
fi
|
||||
drained_slot=
|
||||
wait_for_lb "$new"
|
||||
if [ "$step" -eq 1 ]; then
|
||||
first=$new
|
||||
else
|
||||
second=$new
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$(compose ps -q api | wc -l | tr -d ' ')" -ne 2 ]; then
|
||||
printf '%s\n' 'Expected exactly two running API replicas after replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
completed=1
|
||||
printf '{"replaced_processes":2,"min_ready_during_update":%s}\n' "$min_ready"
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
hl_scripts_dir=$(CDPATH='' cd -- "$(dirname "$0")" && pwd)
|
||||
hl_repo_dir=$(dirname "$hl_scripts_dir")
|
||||
hl_venv_dir="$hl_repo_dir/.venv"
|
||||
"${PYTHON:-python3}" -m venv "$hl_venv_dir"
|
||||
"$hl_venv_dir/bin/python" -m pip install -r "$hl_repo_dir/requirements-dev.txt"
|
||||
"$hl_venv_dir/bin/python" -m pip check
|
||||
printf '%s\n' 'Test dependencies installed into .venv. Run .venv/bin/python student-tests/stack.py verify'
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
hl_scripts_dir=$(dirname "$0")
|
||||
exec "${PYTHON:-python3}" "$hl_scripts_dir/../student-tests/stack.py" start-db
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
hl_scripts_dir=$(dirname "$0")
|
||||
exec "${PYTHON:-python3}" "$hl_scripts_dir/../student-tests/stack.py" start-cache
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
for container_id in $(docker compose --env-file .env ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --timeout 5 "$container_id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf 'API replica %s not found\n' "$TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
hl_scripts_dir=$(dirname "$0")
|
||||
exec "${PYTHON:-python3}" "$hl_scripts_dir/../student-tests/stack.py" stop-db
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
hl_scripts_dir=$(dirname "$0")
|
||||
exec "${PYTHON:-python3}" "$hl_scripts_dir/../student-tests/stack.py" stop-cache
|
||||
Reference in New Issue
Block a user