Files
mtusi2026lab4/scripts/lb_readiness_check.sh
T
2026-10-21 21:22:42 +03:00

195 lines
6.1 KiB
Bash
Executable File

#!/bin/sh
# Проверка критерия «LB исключает неготовую реплику» (lab3).
#
# Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её
# доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и
# получает трафик через LB. После остановки прокси реплика остаётся доступной по
# сети (процесс жив, порт открыт), но /health/ready отвечает 503 — и балансировщик
# обязан убрать её из ротации и не отправлять ей ни одного запроса.
#
# scripts/lb_readiness_check.sh # стенд поднят (make up)
# OUT=evidence/lb-readiness.txt scripts/lb_readiness_check.sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
# .env определяет, на каком порту публикуется LB (compose.yaml) и какие креды у
# хранилищ, поэтому значения берём оттуда же, а не из окружения вызывающего.
if [ -f .env ]; then
set -a
# shellcheck disable=SC1091
. ./.env
set +a
fi
OUT=${OUT:-evidence/lb-readiness.txt}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
NET=${PROJECT}_default
IMAGE=${IMAGE:-mtusi-messenger:lab3}
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
BASE_URL=${BASE_URL:-http://localhost:8080}
CANDIDATE=lb-readiness-candidate
PROXY=lb-readiness-pgproxy
REQUESTS=${REQUESTS:-40}
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
cleanup() {
docker rm -f "$CANDIDATE" "$PROXY" >/dev/null 2>&1 || true
}
trap cleanup EXIT INT TERM
cleanup
instances() {
tries=${1:-$REQUESTS}
i=0
while [ "$i" -lt "$tries" ]; do
curl -fsS --noproxy '*' -D - -o /dev/null "$BASE_URL/health/live" 2>/dev/null \
| awk 'tolower($1) == "x-instance-id:" { print $2 }'
i=$((i + 1))
done | sort | uniq -c
}
stats() {
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ { printf "%s=%s ", $2, $18 }'
}
readiness() {
docker exec "$CANDIDATE" wget -qO- -S -T 3 http://127.0.0.1:8080/health/ready 2>&1 \
| awk '/^ HTTP\//{print $2; exit}'
}
wait_readiness() {
want=$1
attempt=0
while [ "$attempt" -lt 30 ]; do
[ "$(readiness)" = "$want" ] && return 0
attempt=$((attempt + 1))
sleep 1
done
return 1
}
up_slots() {
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }'
}
wait_new_up_slot() {
before=$1
attempt=0
while [ "$attempt" -lt 30 ]; do
for slot in $(up_slots); do
case " $before " in
*" $slot "*) ;;
*) echo "$slot"; return 0 ;;
esac
done
attempt=$((attempt + 1))
sleep 1
done
return 1
}
wait_slot_status() {
slot=$1
want=$2
attempt=0
while [ "$attempt" -lt 60 ]; do
status=$(curl -s --noproxy '*' "$STATS" | awk -F, -v s="$slot" '$1 ~ /^api/ && $2 == s { print $18; exit }')
if [ "$status" = "$want" ]; then
SWAIT_SECONDS=$((attempt / 2))
return 0
fi
attempt=$((attempt + 1))
sleep 0.5
done
return 1
}
{
echo "# LB excludes a not-ready but reachable replica (lab3)"
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)"
echo
echo "## 1. baseline: requests through LB by replica"
instances
baseline_slots=$(up_slots)
echo " stats: $(stats)"
echo
echo "## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias"
docker run -d --name "$PROXY" --network "$NET" alpine/socat@sha256:fb3862c949af7c79996d0e514ee96128644d840835ed5368eeecb14a869de361 \
tcp-listen:5432,fork,reuseaddr tcp-connect:postgres:5432 >/dev/null
docker run -d --name "$CANDIDATE" --network "$NET" --network-alias api-replica \
-e APP_CONFIG=/etc/messenger/dev/config.yaml \
-e POSTGRES_HOST="$PROXY" -e POSTGRES_PORT=5432 \
-e POSTGRES_DATABASE="${POSTGRES_DATABASE:-messenger}" \
-e POSTGRES_USER="${POSTGRES_USER:-messenger}" \
-e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \
-e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \
"$IMAGE" >/dev/null
if ! wait_readiness 200; then
echo " candidate did not become ready" >&2
exit 1
fi
echo " instance-id: $(docker inspect -f '{{.Config.Hostname}}' "$CANDIDATE"), readiness while its database is reachable: 200"
slot=$(wait_new_up_slot "$baseline_slots" || true)
if [ -z "$slot" ]; then
echo " candidate did not appear UP in the load balancer" >&2
exit 1
fi
echo " load balancer slot for the candidate: $slot, stats: $(stats)"
echo
echo "## 3. cut the candidate's database access (stop only the TCP proxy)"
docker stop "$PROXY" >/dev/null
if ! wait_readiness 503; then
echo " candidate readiness did not become 503" >&2
exit 1
fi
echo " candidate is still running: $(docker inspect -f '{{.State.Status}}' "$CANDIDATE"), readiness: 503"
echo " waiting for the load balancer to mark $slot DOWN"
if wait_slot_status "$slot" DOWN; then
echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)"
else
echo " WARNING: $slot is still not DOWN" >&2
fi
echo " stats after failure: $(stats)"
echo
echo "## 4. ${REQUESTS} requests through LB while the candidate is not ready"
instances
echo " (the candidate must not appear above)"
echo
echo "## 5. business writes through LB during the exclusion"
# shellcheck disable=SC2086
$NO_PROXY_ENV python3 tests/load.py --base-url "$BASE_URL" --duration 5 --concurrency 2 \
| grep -E '"successful_rps"|"failed_requests"|"error_ratio"'
echo
echo "## 6. cleanup"
cleanup
sleep 5
echo " stats after cleanup: $(stats)"
} > "$OUT" 2>&1
cat "$OUT"
printf 'lb_readiness_check.sh: evidence in %s\n' "$OUT"