Files
mtusi2026lab3/scripts/collect-evidence.sh
T

163 lines
8.0 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/sh
set -u
cd "$(dirname "$0")/.."
mkdir -p evidence
COMMIT="$(git rev-parse --short HEAD)$(git diff --quiet HEAD -- src prisma test scripts infra compose.yaml Dockerfile || echo ' (dirty)')"
header() {
printf '# %s\n# date: %s\n# commit: %s\n\n' "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$COMMIT"
}
run() {
echo "\$ $*"
"$@" 2>&1 || echo "EXIT CODE $?"
echo
}
started() {
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb)
}
api_states() {
docker inspect -f '{{.Name}} id={{slice .Id 0 12}} {{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' $(docker compose ps -aq api)
}
restore_api() {
for id in $(docker compose ps -aq --status exited api); do
docker start "$id" >/dev/null
done
python3 scripts/probe.py ready >/dev/null
sleep 3
}
echo 'container.txt'
{
header 'Стенд: сервисы, порты, лимиты, секреты, миграции'
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)"
echo '## container_name не задан (одно имя сервиса масштабируется)'
run sh -c "docker compose config | grep -c container_name || true"
echo '## Секреты: каждый сервис видит только свои файлы'
for service in postgres redis api; do
run docker compose exec -T "$service" ls /run/secrets
done
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
run docker compose exec -T lb haproxy -v
run docker compose exec -T postgres postgres --version
run docker compose exec -T redis redis-server --version
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет'
run docker compose run --rm migrate
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at'
} > evidence/container.txt
echo 'tests.txt'
{
header 'Собственные e2e-тесты (PostgreSQL + Redis из compose.test.yaml)'
docker compose -f compose.test.yaml build -q tests
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
docker compose -f compose.test.yaml down >/dev/null 2>&1
} > evidence/tests.txt
echo 'smoke.txt'
{
header 'Публичные проверки через LB; разные реплики обслуживают одну сессию'
run make check
run make test
run sh -c "docker compose ps -q api | cut -c1-12"
run python3 scripts/probe.py instances 6
} > evidence/smoke.txt
echo 'persistence.txt'
{
header 'make test-persistence: restart.sh перезапускает postgres, redis, api, lb с сохранением volumes'
echo '## до'
started
echo
run make test-persistence ACTION_SCRIPT=scripts/restart.sh
echo '## после'
started
} > evidence/persistence.txt
echo 'failover.txt'
{
header 'make test-failover: stop-one.sh останавливает реплику, обслужившую тест; затем вторую'
for round in 1 2; do
echo "## Прогон $round"
echo '### до'
api_states
echo
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
run make test-failover ACTION_SCRIPT=scripts/stop-one.sh
echo '### после: остановленная реплика Exited (0) — штатное завершение по SIGTERM'
api_states
echo
echo '### журнал HAProxy'
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP|maintenance|READY' | grep -v '^\[WARNING\]'
echo
restore_api
python3 scripts/probe.py status GET /health/live >/dev/null
done
echo '## Проверка guard: stop-one.sh отказывается останавливать не api'
run env TARGET_INSTANCE_ID="$(docker compose ps -q postgres | cut -c1-12)" sh scripts/stop-one.sh
} > evidence/failover.txt
echo 'experiments: idempotency, failover, failover --kill, rolling'
python3 scripts/experiment.py idempotency --output evidence/idempotency.json >/dev/null 2>&1
python3 scripts/experiment.py failover --output evidence/failover-stop.json >/dev/null 2>&1
python3 scripts/experiment.py failover --kill --output evidence/failover-kill.json >/dev/null 2>&1
docker compose build -q api >/dev/null 2>&1
python3 scripts/experiment.py rolling --output evidence/rolling.json >/dev/null 2>&1
echo 'failures.txt'
{
header 'Отказы зависимостей, ошибки LB, expiry'
echo '## 1. Redis остановлен: вход и Bearer через PostgreSQL; logout — 503 без отзыва (токен остаётся действующим)'
run docker compose stop redis
run python3 scripts/probe.py logout
run docker compose start redis
sleep 3
echo '## 2. Redis снова доступен: logout отзывает токен на всех репликах'
run python3 scripts/probe.py logout
echo '## 3. PostgreSQL остановлен: live 200, запрос с токеном — 503 без внутренних деталей, аноним — 401'
run docker compose stop postgres
run python3 scripts/probe.py status GET /health/live
run python3 scripts/probe.py status GET /api/v1/chats 'Bearer unknown-token'
run python3 scripts/probe.py status GET /api/v1/users/me
run docker compose start postgres
run python3 scripts/probe.py ready
echo '## 4. Все реплики API остановлены: LB отвечает 503 в формате ошибки контракта'
run docker compose stop api
run python3 scripts/probe.py status GET /api/v1/users/me
run docker compose start api
run python3 scripts/probe.py ready
echo '## 5. Живая, но неготовая реплика: PostgreSQL отклоняет подключения только от одной API (pg_hba reject)'
target=$(docker compose ps -q api | head -1 | cut -c1-12)
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$target")
hba=/var/lib/postgresql/data/pg_hba.conf
echo "target=$target ip=$ip"
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba"
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null
sleep 4
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')"
run python3 scripts/probe.py instances 10
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]'
echo
docker compose exec -T postgres sh -c "cp /tmp/pg_hba.bak $hba && chown postgres:postgres $hba && chmod 600 $hba"
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf()" >/dev/null
sleep 4
echo '### доступ возвращён: реплика снова готова и получает запросы'
run python3 scripts/probe.py instances 6
echo '## 6. Expiry: реплики пересозданы с SESSION_TTL_SECONDS=2'
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --wait api >/dev/null 2>&1
run sh -c "docker exec \$(docker compose ps -q api | head -1) printenv SESSION_TTL_SECONDS"
run python3 scripts/probe.py expiry 2
docker compose up -d --no-deps --wait api >/dev/null 2>&1
run python3 scripts/probe.py ready
} > evidence/failures.txt
echo 'done'