Fixes for mandatory level 3: 1. Reduce nginx proxy_connect_timeout to 1s for faster failover detection - Prevents 499 errors during failover (client timeout < connect timeout) 2. Set executable permissions on scripts (git update-index --chmod=+x) - restart.sh, stop-one.sh now 100755 3. Fix REPORT.md errors: - Correct test command: make test-failover (not py -3 tests\failover.py) - Fix health check explanation (nginx uses timeouts, not Docker healthcheck) - Fix line numbers: INSTANCE_ID=39, X-Instance-Id=88, message()=321 - Add note about nginx warmup period (~10s after startup) Additions for level 4: 4. Add scripts/measure-failover.sh - measure RPS, errors, recovery time - Sequential testing of api-1 and api-2 stops - Baseline, failover, and recovery measurements 5. Add scripts/rolling-update.sh - demonstrate rolling update under load - Generates load for 60s while updating replicas sequentially - Measures errors during rolling update 6. Update REPORT.md with sections 10-11 for level 4 requirements All changes address reviewer feedback and ensure tests pass consistently. Co-Authored-By: Claude <noreply@anthropic.com>
131 lines
3.4 KiB
Bash
Executable File
131 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# Скрипт демонстрации rolling update под нагрузкой.
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
|
OUTPUT_DIR="${OUTPUT_DIR:-./rolling_results}"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
echo "=== Демонстрация rolling update под нагрузкой ==="
|
|
echo ""
|
|
|
|
# Функция для генерации нагрузки в фоне
|
|
generate_load() {
|
|
local output_file=$1
|
|
local duration=${2:-60}
|
|
|
|
echo "Запуск генератора нагрузки (${duration}s)..."
|
|
|
|
local success=0
|
|
local errors=0
|
|
local start_time=$(date +%s)
|
|
local end_time=$((start_time + duration))
|
|
|
|
while [ $(date +%s) -lt $end_time ]; do
|
|
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/health/ready" 2>/dev/null || echo "000")
|
|
http_code=$(echo "$response" | tail -1)
|
|
|
|
if [ "$http_code" = "200" ]; then
|
|
success=$((success + 1))
|
|
else
|
|
errors=$((errors + 1))
|
|
echo "$(date +%s.%N) ERROR $http_code" >> "$output_file.errors"
|
|
fi
|
|
|
|
sleep 0.1
|
|
done
|
|
|
|
local actual_duration=$(($(date +%s) - start_time))
|
|
local rps=$(echo "scale=2; $success / $actual_duration" | bc)
|
|
local error_rate=$(echo "scale=2; ($errors * 100) / ($success + $errors)" | bc)
|
|
|
|
cat > "$output_file" <<EOF
|
|
Успешных запросов: $success
|
|
Ошибок: $errors
|
|
Длительность: ${actual_duration}s
|
|
RPS: $rps
|
|
Процент ошибок: ${error_rate}%
|
|
EOF
|
|
|
|
echo "Нагрузка завершена: $success успешных, $errors ошибок (RPS: $rps)"
|
|
}
|
|
|
|
# Проверка исходного состояния
|
|
echo "1. Проверка текущего состояния"
|
|
docker compose ps
|
|
echo ""
|
|
|
|
# Запуск генератора нагрузки в фоне
|
|
LOAD_OUTPUT="$OUTPUT_DIR/load_during_rolling_$(date +%Y%m%d_%H%M%S).txt"
|
|
generate_load "$LOAD_OUTPUT" 60 &
|
|
LOAD_PID=$!
|
|
|
|
sleep 5
|
|
|
|
# Rolling update: поочерёдное пересоздание реплик
|
|
echo ""
|
|
echo "2. Rolling update api-1"
|
|
echo "Останавливаем api-1..."
|
|
docker compose stop api-1
|
|
|
|
sleep 2
|
|
|
|
echo "Пересоздаём api-1..."
|
|
docker compose up -d --no-deps --build api-1
|
|
|
|
echo "Ожидаем готовности api-1 (health check)..."
|
|
for i in {1..30}; do
|
|
if docker compose ps api-1 | grep -q "healthy"; then
|
|
echo "api-1 готов!"
|
|
break
|
|
fi
|
|
echo "Попытка $i/30..."
|
|
sleep 2
|
|
done
|
|
|
|
echo ""
|
|
echo "3. Rolling update api-2"
|
|
echo "Останавливаем api-2..."
|
|
docker compose stop api-2
|
|
|
|
sleep 2
|
|
|
|
echo "Пересоздаём api-2..."
|
|
docker compose up -d --no-deps --build api-2
|
|
|
|
echo "Ожидаем готовности api-2 (health check)..."
|
|
for i in {1..30}; do
|
|
if docker compose ps api-2 | grep -q "healthy"; then
|
|
echo "api-2 готов!"
|
|
break
|
|
fi
|
|
echo "Попытка $i/30..."
|
|
sleep 2
|
|
done
|
|
|
|
echo ""
|
|
echo "4. Ожидание завершения генератора нагрузки"
|
|
wait $LOAD_PID
|
|
|
|
echo ""
|
|
echo "5. Финальное состояние"
|
|
docker compose ps
|
|
|
|
echo ""
|
|
echo "=== Результаты rolling update ==="
|
|
cat "$LOAD_OUTPUT"
|
|
|
|
if [ -f "$LOAD_OUTPUT.errors" ]; then
|
|
ERROR_COUNT=$(wc -l < "$LOAD_OUTPUT.errors")
|
|
echo ""
|
|
echo "Зафиксировано ошибок: $ERROR_COUNT"
|
|
echo "Первые 10 ошибок:"
|
|
head -10 "$LOAD_OUTPUT.errors"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Rolling update завершён. Результаты в: $OUTPUT_DIR"
|