Fixes for mandatory level 3: 1. Reduce nginx proxy_connect_timeout to 1s for faster failover detection - Prevents 499 errors during failover (client timeout < connect timeout) 2. Set executable permissions on scripts (git update-index --chmod=+x) - restart.sh, stop-one.sh now 100755 3. Fix REPORT.md errors: - Correct test command: make test-failover (not py -3 tests\failover.py) - Fix health check explanation (nginx uses timeouts, not Docker healthcheck) - Fix line numbers: INSTANCE_ID=39, X-Instance-Id=88, message()=321 - Add note about nginx warmup period (~10s after startup) Additions for level 4: 4. Add scripts/measure-failover.sh - measure RPS, errors, recovery time - Sequential testing of api-1 and api-2 stops - Baseline, failover, and recovery measurements 5. Add scripts/rolling-update.sh - demonstrate rolling update under load - Generates load for 60s while updating replicas sequentially - Measures errors during rolling update 6. Update REPORT.md with sections 10-11 for level 4 requirements All changes address reviewer feedback and ensure tests pass consistently. Co-Authored-By: Claude <noreply@anthropic.com>
138 lines
3.8 KiB
Bash
Executable File
138 lines
3.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Скрипт измерения RPS, ошибок и времени восстановления при failover.
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
|
DURATION="${DURATION:-30}"
|
|
OUTPUT_DIR="${OUTPUT_DIR:-./failover_results}"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
echo "=== Измерение производительности при failover ==="
|
|
echo "Base URL: $BASE_URL"
|
|
echo "Duration: ${DURATION}s"
|
|
echo ""
|
|
|
|
# Функция для измерения RPS
|
|
measure_rps() {
|
|
local label=$1
|
|
local output_file="$OUTPUT_DIR/${label}_$(date +%Y%m%d_%H%M%S).txt"
|
|
|
|
echo "Измерение: $label"
|
|
|
|
# Счётчики
|
|
local success=0
|
|
local errors=0
|
|
local start_time=$(date +%s)
|
|
local end_time=$((start_time + DURATION))
|
|
|
|
while [ $(date +%s) -lt $end_time ]; do
|
|
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/health/ready" 2>/dev/null || echo "000")
|
|
http_code=$(echo "$response" | tail -1)
|
|
|
|
if [ "$http_code" = "200" ]; then
|
|
success=$((success + 1))
|
|
else
|
|
errors=$((errors + 1))
|
|
fi
|
|
|
|
sleep 0.1
|
|
done
|
|
|
|
local actual_duration=$(($(date +%s) - start_time))
|
|
local rps=$(echo "scale=2; $success / $actual_duration" | bc)
|
|
|
|
cat > "$output_file" <<EOF
|
|
Тест: $label
|
|
Длительность: ${actual_duration}s
|
|
Успешных запросов: $success
|
|
Ошибок: $errors
|
|
RPS: $rps
|
|
Процент ошибок: $(echo "scale=2; ($errors * 100) / ($success + $errors)" | bc)%
|
|
EOF
|
|
|
|
cat "$output_file"
|
|
echo ""
|
|
}
|
|
|
|
# Baseline измерение
|
|
echo "1. Baseline (обе реплики работают)"
|
|
measure_rps "baseline"
|
|
|
|
# Остановка api-1
|
|
echo "2. Остановка api-1"
|
|
START_STOP=$(date +%s.%N)
|
|
docker compose stop api-1
|
|
END_STOP=$(date +%s.%N)
|
|
STOP_TIME=$(echo "$END_STOP - $START_STOP" | bc)
|
|
echo "Время остановки: ${STOP_TIME}s"
|
|
echo ""
|
|
|
|
sleep 2
|
|
|
|
echo "3. Измерение после остановки api-1 (только api-2)"
|
|
measure_rps "after_stop_api1"
|
|
|
|
# Восстановление api-1
|
|
echo "4. Восстановление api-1"
|
|
START_RESTART=$(date +%s.%N)
|
|
docker compose start api-1
|
|
sleep 15 # Ждём health check
|
|
END_RESTART=$(date +%s.%N)
|
|
RESTART_TIME=$(echo "$END_RESTART - $START_RESTART" | bc)
|
|
echo "Время восстановления: ${RESTART_TIME}s"
|
|
echo ""
|
|
|
|
echo "5. Измерение после восстановления api-1"
|
|
measure_rps "after_restart_api1"
|
|
|
|
# Остановка api-2
|
|
echo "6. Остановка api-2"
|
|
START_STOP2=$(date +%s.%N)
|
|
docker compose stop api-2
|
|
END_STOP2=$(date +%s.%N)
|
|
STOP_TIME2=$(echo "$END_STOP2 - $START_STOP2" | bc)
|
|
echo "Время остановки: ${STOP_TIME2}s"
|
|
echo ""
|
|
|
|
sleep 2
|
|
|
|
echo "7. Измерение после остановки api-2 (только api-1)"
|
|
measure_rps "after_stop_api2"
|
|
|
|
# Восстановление api-2
|
|
echo "8. Восстановление api-2"
|
|
START_RESTART2=$(date +%s.%N)
|
|
docker compose start api-2
|
|
sleep 15
|
|
END_RESTART2=$(date +%s.%N)
|
|
RESTART_TIME2=$(echo "$END_RESTART2 - $START_RESTART2" | bc)
|
|
echo "Время восстановления: ${RESTART_TIME2}s"
|
|
echo ""
|
|
|
|
echo "9. Финальное измерение (обе реплики)"
|
|
measure_rps "final"
|
|
|
|
# Генерация итогового отчёта
|
|
cat > "$OUTPUT_DIR/summary.txt" <<EOF
|
|
=== Итоговый отчёт по failover ===
|
|
|
|
Остановка api-1:
|
|
- Время остановки: ${STOP_TIME}s
|
|
- Время восстановления: ${RESTART_TIME}s
|
|
|
|
Остановка api-2:
|
|
- Время остановки: ${STOP_TIME2}s
|
|
- Время восстановления: ${RESTART_TIME2}s
|
|
|
|
Детальные результаты см. в файлах ${OUTPUT_DIR}/*.txt
|
|
EOF
|
|
|
|
cat "$OUTPUT_DIR/summary.txt"
|
|
|
|
echo ""
|
|
echo "=== Измерения завершены ==="
|
|
echo "Результаты в: $OUTPUT_DIR"
|