195 lines
6.1 KiB
Bash
Executable File
195 lines
6.1 KiB
Bash
Executable File
#!/bin/sh
|
|
# Проверка критерия «LB исключает неготовую реплику» (lab3).
|
|
#
|
|
# Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её
|
|
# доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и
|
|
# получает трафик через LB. После остановки прокси реплика остаётся доступной по
|
|
# сети (процесс жив, порт открыт), но /health/ready отвечает 503 — и балансировщик
|
|
# обязан убрать её из ротации и не отправлять ей ни одного запроса.
|
|
#
|
|
# scripts/lb_readiness_check.sh # стенд поднят (make up)
|
|
# OUT=evidence/lb-readiness.txt scripts/lb_readiness_check.sh
|
|
set -eu
|
|
|
|
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
cd "$ROOT_DIR"
|
|
|
|
# .env определяет, на каком порту публикуется LB (compose.yaml) и какие креды у
|
|
# хранилищ, поэтому значения берём оттуда же, а не из окружения вызывающего.
|
|
if [ -f .env ]; then
|
|
set -a
|
|
# shellcheck disable=SC1091
|
|
. ./.env
|
|
set +a
|
|
fi
|
|
|
|
OUT=${OUT:-evidence/lb-readiness.txt}
|
|
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
|
NET=${PROJECT}_default
|
|
IMAGE=${IMAGE:-mtusi-messenger:lab3}
|
|
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
|
|
BASE_URL=${BASE_URL:-http://localhost:8080}
|
|
CANDIDATE=lb-readiness-candidate
|
|
PROXY=lb-readiness-pgproxy
|
|
REQUESTS=${REQUESTS:-40}
|
|
|
|
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
|
|
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
|
|
|
|
cleanup() {
|
|
docker rm -f "$CANDIDATE" "$PROXY" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
trap cleanup EXIT INT TERM
|
|
cleanup
|
|
|
|
instances() {
|
|
tries=${1:-$REQUESTS}
|
|
i=0
|
|
|
|
while [ "$i" -lt "$tries" ]; do
|
|
curl -fsS --noproxy '*' -D - -o /dev/null "$BASE_URL/health/live" 2>/dev/null \
|
|
| awk 'tolower($1) == "x-instance-id:" { print $2 }'
|
|
i=$((i + 1))
|
|
done | sort | uniq -c
|
|
}
|
|
|
|
stats() {
|
|
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ { printf "%s=%s ", $2, $18 }'
|
|
}
|
|
|
|
readiness() {
|
|
docker exec "$CANDIDATE" wget -qO- -S -T 3 http://127.0.0.1:8080/health/ready 2>&1 \
|
|
| awk '/^ HTTP\//{print $2; exit}'
|
|
}
|
|
|
|
wait_readiness() {
|
|
want=$1
|
|
attempt=0
|
|
|
|
while [ "$attempt" -lt 30 ]; do
|
|
[ "$(readiness)" = "$want" ] && return 0
|
|
attempt=$((attempt + 1))
|
|
sleep 1
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
up_slots() {
|
|
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }'
|
|
}
|
|
|
|
wait_new_up_slot() {
|
|
before=$1
|
|
attempt=0
|
|
|
|
while [ "$attempt" -lt 30 ]; do
|
|
for slot in $(up_slots); do
|
|
case " $before " in
|
|
*" $slot "*) ;;
|
|
*) echo "$slot"; return 0 ;;
|
|
esac
|
|
done
|
|
|
|
attempt=$((attempt + 1))
|
|
sleep 1
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
wait_slot_status() {
|
|
slot=$1
|
|
want=$2
|
|
attempt=0
|
|
|
|
while [ "$attempt" -lt 60 ]; do
|
|
status=$(curl -s --noproxy '*' "$STATS" | awk -F, -v s="$slot" '$1 ~ /^api/ && $2 == s { print $18; exit }')
|
|
|
|
if [ "$status" = "$want" ]; then
|
|
SWAIT_SECONDS=$((attempt / 2))
|
|
return 0
|
|
fi
|
|
|
|
attempt=$((attempt + 1))
|
|
sleep 0.5
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
{
|
|
echo "# LB excludes a not-ready but reachable replica (lab3)"
|
|
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)"
|
|
echo
|
|
echo "## 1. baseline: requests through LB by replica"
|
|
instances
|
|
baseline_slots=$(up_slots)
|
|
echo " stats: $(stats)"
|
|
echo
|
|
echo "## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias"
|
|
docker run -d --name "$PROXY" --network "$NET" alpine/socat@sha256:fb3862c949af7c79996d0e514ee96128644d840835ed5368eeecb14a869de361 \
|
|
tcp-listen:5432,fork,reuseaddr tcp-connect:postgres:5432 >/dev/null
|
|
docker run -d --name "$CANDIDATE" --network "$NET" --network-alias api-replica \
|
|
-e APP_CONFIG=/etc/messenger/dev/config.yaml \
|
|
-e POSTGRES_HOST="$PROXY" -e POSTGRES_PORT=5432 \
|
|
-e POSTGRES_DATABASE="${POSTGRES_DATABASE:-messenger}" \
|
|
-e POSTGRES_USER="${POSTGRES_USER:-messenger}" \
|
|
-e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \
|
|
-e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \
|
|
"$IMAGE" >/dev/null
|
|
|
|
if ! wait_readiness 200; then
|
|
echo " candidate did not become ready" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo " instance-id: $(docker inspect -f '{{.Config.Hostname}}' "$CANDIDATE"), readiness while its database is reachable: 200"
|
|
|
|
slot=$(wait_new_up_slot "$baseline_slots" || true)
|
|
if [ -z "$slot" ]; then
|
|
echo " candidate did not appear UP in the load balancer" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo " load balancer slot for the candidate: $slot, stats: $(stats)"
|
|
echo
|
|
echo "## 3. cut the candidate's database access (stop only the TCP proxy)"
|
|
docker stop "$PROXY" >/dev/null
|
|
|
|
if ! wait_readiness 503; then
|
|
echo " candidate readiness did not become 503" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo " candidate is still running: $(docker inspect -f '{{.State.Status}}' "$CANDIDATE"), readiness: 503"
|
|
echo " waiting for the load balancer to mark $slot DOWN"
|
|
|
|
if wait_slot_status "$slot" DOWN; then
|
|
echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)"
|
|
else
|
|
echo " WARNING: $slot is still not DOWN" >&2
|
|
fi
|
|
|
|
echo " stats after failure: $(stats)"
|
|
echo
|
|
echo "## 4. ${REQUESTS} requests through LB while the candidate is not ready"
|
|
instances
|
|
echo " (the candidate must not appear above)"
|
|
echo
|
|
echo "## 5. business writes through LB during the exclusion"
|
|
# shellcheck disable=SC2086
|
|
$NO_PROXY_ENV python3 tests/load.py --base-url "$BASE_URL" --duration 5 --concurrency 2 \
|
|
| grep -E '"successful_rps"|"failed_requests"|"error_ratio"'
|
|
echo
|
|
echo "## 6. cleanup"
|
|
cleanup
|
|
sleep 5
|
|
echo " stats after cleanup: $(stats)"
|
|
} > "$OUT" 2>&1
|
|
|
|
cat "$OUT"
|
|
printf 'lb_readiness_check.sh: evidence in %s\n' "$OUT"
|