#!/bin/sh set -eu cd "$(dirname "$0")/.." if [ -f .env ]; then compose() { docker compose --env-file .env "$@"; } else : "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}" compose() { docker compose "$@"; } fi base_url=${BASE_URL:-http://127.0.0.1:8080} base_url=${base_url%/} container_ip() { docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" } haproxy_cli() { docker exec "$lb_container" sh -c \ 'printf "%s\n" "$1" | socat - TCP4:127.0.0.1:9999' sh "$1" } slot_for_ip() { haproxy_cli 'show servers state api_replicas' | awk -v address="$1" '$2 == "api_replicas" && $5 == address { print $4; exit }' } ip_for_slot() { haproxy_cli 'show servers state api_replicas' | awk -v name="$1" '$2 == "api_replicas" && $4 == name { print $5; exit }' } set_server_state() { response=$(haproxy_cli "set server api_replicas/$1 state $2") if [ -n "$response" ]; then printf 'HAProxy rejected state change for %s: %s\n' "$1" "$response" >&2 return 1 fi } wait_drained() { attempts=0 while [ "$attempts" -lt 30 ]; do active=$(haproxy_cli 'show stat' | awk -F, -v name="$1" '$1 == "api_replicas" && $2 == name { print $5; exit }') if [ "$active" = 0 ]; then return 0 fi attempts=$((attempts + 1)) sleep 1 done printf 'HAProxy backend %s did not drain\n' "$1" >&2 return 1 } wait_lb_slot_for_ip() { attempts=0 while [ "$attempts" -lt 30 ]; do found=$(slot_for_ip "$1") if [ -n "$found" ]; then printf '%s\n' "$found" return 0 fi attempts=$((attempts + 1)) sleep 1 done printf 'HAProxy did not resolve new API address %s\n' "$1" >&2 return 1 } wait_old_address_released() { attempts=0 while [ "$attempts" -lt 30 ]; do if [ "$(ip_for_slot "$1")" != "$2" ]; then return 0 fi attempts=$((attempts + 1)) sleep 1 done printf 'HAProxy kept retired API address %s on %s\n' "$2" "$1" >&2 return 1 } container_except() { excluded=$1 for candidate in $(compose ps -aq api); do if [ "$candidate" != "$excluded" ]; then printf '%s\n' "$candidate" return 0 fi done return 1 } is_ready() { docker exec "$1" wget -q -O /dev/null http://127.0.0.1:8080/health/ready >/dev/null 2>&1 } ready_count() { count=0 for candidate in $(compose ps -q api); do if is_ready "$candidate"; then count=$((count + 1)) fi done printf '%s\n' "$count" } wait_ready() { candidate=$1 attempts=0 while [ "$attempts" -lt 30 ]; do if is_ready "$candidate"; then return 0 fi attempts=$((attempts + 1)) sleep 1 done printf 'API container did not become ready: %s\n' "$candidate" >&2 return 1 } wait_for_lb() { candidate=$1 expected=$(docker inspect --format '{{.Config.Hostname}}' "$candidate") attempts=0 while [ "$attempts" -lt 30 ]; do observed=$(curl -sS --max-time 2 -D - -o /dev/null "$base_url/health/ready" 2>/dev/null | awk 'tolower($1) == "x-instance-id:" { gsub("\r", "", $2); print $2; exit }') if [ "$observed" = "$expected" ]; then return 0 fi attempts=$((attempts + 1)) sleep 1 done printf 'HAProxy did not route to new API container: %s\n' "$candidate" >&2 return 1 } set -- $(compose ps -q api) if [ "$#" -ne 2 ] || ! is_ready "$1" || ! is_ready "$2"; then printf '%s\n' 'Two ready API replicas are required before replacement' >&2 exit 1 fi first=$1 second=$2 lb_container=$(compose ps -q lb) if [ -z "$lb_container" ] || [ -z "$(slot_for_ip "$(container_ip "$first")")" ] || [ -z "$(slot_for_ip "$(container_ip "$second")")" ]; then printf '%s\n' 'HAProxy Runtime API did not expose both ready API backends' >&2 exit 1 fi min_ready=2 completed=0 drained_slot= restore_on_failure() { if [ "$completed" -ne 1 ]; then compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2 || true if [ -n "$drained_slot" ]; then set_server_state "$drained_slot" ready >&2 || true fi fi } trap restore_on_failure 0 for step in 1 2; do if [ "$step" -eq 1 ]; then old=$first survivor=$second else old=$second survivor=$first fi if [ "$(container_except "$survivor")" != "$old" ] || ! is_ready "$survivor"; then printf 'API replica %s changed or survivor is not ready\n' "$step" >&2 exit 1 fi old_ip=$(container_ip "$old") old_slot=$(slot_for_ip "$old_ip") if [ -z "$old_slot" ]; then printf 'HAProxy backend for %s not found\n' "$old" >&2 exit 1 fi set_server_state "$old_slot" drain drained_slot=$old_slot wait_drained "$old_slot" sleep 1 docker stop --timeout 18 "$old" >/dev/null current_ready=$(ready_count) if [ "$current_ready" -lt 1 ]; then printf '%s\n' 'No ready API replica remained during replacement' >&2 exit 1 fi if [ "$current_ready" -lt "$min_ready" ]; then min_ready=$current_ready fi docker rm "$old" >/dev/null compose up -d --no-deps --no-build --no-recreate --scale api=2 api >&2 new=$(container_except "$survivor") if [ "$new" = "$old" ] || ! is_ready "$survivor"; then printf 'Compose unexpectedly replaced another API replica at step %s\n' "$step" >&2 exit 1 fi wait_ready "$new" new_ip=$(container_ip "$new") new_slot=$(wait_lb_slot_for_ip "$new_ip") if [ "$new_ip" != "$old_ip" ]; then wait_old_address_released "$old_slot" "$old_ip" fi set_server_state "$new_slot" ready if [ "$new_slot" != "$old_slot" ]; then set_server_state "$old_slot" ready fi drained_slot= wait_for_lb "$new" if [ "$step" -eq 1 ]; then first=$new else second=$new fi done if [ "$(compose ps -q api | wc -l | tr -d ' ')" -ne 2 ]; then printf '%s\n' 'Expected exactly two running API replicas after replacement' >&2 exit 1 fi completed=1 printf '{"replaced_processes":2,"min_ready_during_update":%s}\n' "$min_ready"