forked from mtusi2026/mtusi2026lab3
163 lines
8.0 KiB
Bash
Executable File
163 lines
8.0 KiB
Bash
Executable File
#!/bin/sh
|
||
set -u
|
||
cd "$(dirname "$0")/.."
|
||
mkdir -p evidence
|
||
|
||
COMMIT="$(git rev-parse --short HEAD)$(git diff --quiet HEAD -- src prisma test scripts infra compose.yaml Dockerfile || echo ' (dirty)')"
|
||
|
||
header() {
|
||
printf '# %s\n# date: %s\n# commit: %s\n\n' "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$COMMIT"
|
||
}
|
||
|
||
run() {
|
||
echo "\$ $*"
|
||
"$@" 2>&1 || echo "EXIT CODE $?"
|
||
echo
|
||
}
|
||
|
||
started() {
|
||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb)
|
||
}
|
||
|
||
api_states() {
|
||
docker inspect -f '{{.Name}} id={{slice .Id 0 12}} {{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' $(docker compose ps -aq api)
|
||
}
|
||
|
||
restore_api() {
|
||
for id in $(docker compose ps -aq --status exited api); do
|
||
docker start "$id" >/dev/null
|
||
done
|
||
python3 scripts/probe.py ready >/dev/null
|
||
sleep 3
|
||
}
|
||
|
||
echo 'container.txt'
|
||
{
|
||
header 'Стенд: сервисы, порты, лимиты, секреты, миграции'
|
||
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
|
||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)"
|
||
echo '## container_name не задан (одно имя сервиса масштабируется)'
|
||
run sh -c "docker compose config | grep -c container_name || true"
|
||
echo '## Секреты: каждый сервис видит только свои файлы'
|
||
for service in postgres redis api; do
|
||
run docker compose exec -T "$service" ls /run/secrets
|
||
done
|
||
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
|
||
run docker compose exec -T lb haproxy -v
|
||
run docker compose exec -T postgres postgres --version
|
||
run docker compose exec -T redis redis-server --version
|
||
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет'
|
||
run docker compose run --rm migrate
|
||
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
|
||
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at'
|
||
} > evidence/container.txt
|
||
|
||
echo 'tests.txt'
|
||
{
|
||
header 'Собственные e2e-тесты (PostgreSQL + Redis из compose.test.yaml)'
|
||
docker compose -f compose.test.yaml build -q tests
|
||
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
|
||
docker compose -f compose.test.yaml down >/dev/null 2>&1
|
||
} > evidence/tests.txt
|
||
|
||
echo 'smoke.txt'
|
||
{
|
||
header 'Публичные проверки через LB; разные реплики обслуживают одну сессию'
|
||
run make check
|
||
run make test
|
||
run sh -c "docker compose ps -q api | cut -c1-12"
|
||
run python3 scripts/probe.py instances 6
|
||
} > evidence/smoke.txt
|
||
|
||
echo 'persistence.txt'
|
||
{
|
||
header 'make test-persistence: restart.sh перезапускает postgres, redis, api, lb с сохранением volumes'
|
||
echo '## до'
|
||
started
|
||
echo
|
||
run make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||
echo '## после'
|
||
started
|
||
} > evidence/persistence.txt
|
||
|
||
echo 'failover.txt'
|
||
{
|
||
header 'make test-failover: stop-one.sh останавливает реплику, обслужившую тест; затем вторую'
|
||
for round in 1 2; do
|
||
echo "## Прогон $round"
|
||
echo '### до'
|
||
api_states
|
||
echo
|
||
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||
run make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||
echo '### после: остановленная реплика Exited (0) — штатное завершение по SIGTERM'
|
||
api_states
|
||
echo
|
||
echo '### журнал HAProxy'
|
||
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP|maintenance|READY' | grep -v '^\[WARNING\]'
|
||
echo
|
||
restore_api
|
||
python3 scripts/probe.py status GET /health/live >/dev/null
|
||
done
|
||
echo '## Проверка guard: stop-one.sh отказывается останавливать не api'
|
||
run env TARGET_INSTANCE_ID="$(docker compose ps -q postgres | cut -c1-12)" sh scripts/stop-one.sh
|
||
} > evidence/failover.txt
|
||
|
||
echo 'experiments: idempotency, failover, failover --kill, rolling'
|
||
python3 scripts/experiment.py idempotency --output evidence/idempotency.json >/dev/null 2>&1
|
||
python3 scripts/experiment.py failover --output evidence/failover-stop.json >/dev/null 2>&1
|
||
python3 scripts/experiment.py failover --kill --output evidence/failover-kill.json >/dev/null 2>&1
|
||
docker compose build -q api >/dev/null 2>&1
|
||
python3 scripts/experiment.py rolling --output evidence/rolling.json >/dev/null 2>&1
|
||
|
||
echo 'failures.txt'
|
||
{
|
||
header 'Отказы зависимостей, ошибки LB, expiry'
|
||
echo '## 1. Redis остановлен: вход и Bearer через PostgreSQL; logout — 503 без отзыва (токен остаётся действующим)'
|
||
run docker compose stop redis
|
||
run python3 scripts/probe.py logout
|
||
run docker compose start redis
|
||
sleep 3
|
||
echo '## 2. Redis снова доступен: logout отзывает токен на всех репликах'
|
||
run python3 scripts/probe.py logout
|
||
echo '## 3. PostgreSQL остановлен: live 200, запрос с токеном — 503 без внутренних деталей, аноним — 401'
|
||
run docker compose stop postgres
|
||
run python3 scripts/probe.py status GET /health/live
|
||
run python3 scripts/probe.py status GET /api/v1/chats 'Bearer unknown-token'
|
||
run python3 scripts/probe.py status GET /api/v1/users/me
|
||
run docker compose start postgres
|
||
run python3 scripts/probe.py ready
|
||
echo '## 4. Все реплики API остановлены: LB отвечает 503 в формате ошибки контракта'
|
||
run docker compose stop api
|
||
run python3 scripts/probe.py status GET /api/v1/users/me
|
||
run docker compose start api
|
||
run python3 scripts/probe.py ready
|
||
echo '## 5. Живая, но неготовая реплика: PostgreSQL отклоняет подключения только от одной API (pg_hba reject)'
|
||
target=$(docker compose ps -q api | head -1 | cut -c1-12)
|
||
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$target")
|
||
hba=/var/lib/postgresql/data/pg_hba.conf
|
||
echo "target=$target ip=$ip"
|
||
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||
docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba"
|
||
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null
|
||
sleep 4
|
||
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')"
|
||
run python3 scripts/probe.py instances 10
|
||
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]'
|
||
echo
|
||
docker compose exec -T postgres sh -c "cp /tmp/pg_hba.bak $hba && chown postgres:postgres $hba && chmod 600 $hba"
|
||
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf()" >/dev/null
|
||
sleep 4
|
||
echo '### доступ возвращён: реплика снова готова и получает запросы'
|
||
run python3 scripts/probe.py instances 6
|
||
echo '## 6. Expiry: реплики пересозданы с SESSION_TTL_SECONDS=2'
|
||
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --wait api >/dev/null 2>&1
|
||
run sh -c "docker exec \$(docker compose ps -q api | head -1) printenv SESSION_TTL_SECONDS"
|
||
run python3 scripts/probe.py expiry 2
|
||
docker compose up -d --no-deps --wait api >/dev/null 2>&1
|
||
run python3 scripts/probe.py ready
|
||
} > evidence/failures.txt
|
||
|
||
echo 'done'
|