diff --git a/Makefile b/Makefile index 5439667f..44a153e9 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ - secrets up down reset-data migrate ps logs restore lb-check \ + secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ persistence-evidence storage-evidence cache-evidence test-faults seed explain help: @@ -86,6 +86,12 @@ restore: lb-check: $(PYTHON) scripts/probe.py check --out evidence/lb-check.json +failover-evidence: + sh scripts/failover-evidence.sh "$(BASE_URL)" + +stack-evidence: + sh scripts/evidence-stack.sh + persistence-evidence: sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt cat evidence/persistence.txt diff --git a/openspec/changes/add-lab3-load-balanced-replicas/tasks.md b/openspec/changes/add-lab3-load-balanced-replicas/tasks.md index d015e1fa..2e52a9ac 100644 --- a/openspec/changes/add-lab3-load-balanced-replicas/tasks.md +++ b/openspec/changes/add-lab3-load-balanced-replicas/tasks.md @@ -33,7 +33,7 @@ - [x] 6.1 `scripts/stop-one.sh`, `scripts/restart.sh` (вместо версии лабы 2), `scripts/restore.sh`, `scripts/lb-wait.sh` - [x] 6.2 `scripts/probe.py` (`check`), ruff для `scripts/*.py` -- [ ] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен +- [x] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен - [ ] 6.4 Проверка группы: `make test`, `make lb-check`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `scripts/restore.sh` между ними, `make test-persistence ACTION_SCRIPT=scripts/restart.sh` ## 7. Отчёт diff --git a/scripts/evidence-stack.sh b/scripts/evidence-stack.sh new file mode 100755 index 00000000..4992d1fb --- /dev/null +++ b/scripts/evidence-stack.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +base=${1:-http://$(docker compose port lb 8080)} +stats=http://$(docker compose port lb 8404)/stats\;csv +header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; } +{ + header + echo "\$ docker compose --env-file .env config --quiet && echo valid" + docker compose --env-file .env config --quiet && echo valid + echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'" + docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}' + echo "\$ docker compose config | grep -c container_name" + docker compose config | grep -c container_name || true + echo "\$ docker compose port api 8080 # реплики не публикуют порт" + docker compose port api 8080 2>&1 || true + echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти" + for id in $(docker compose ps -q); do + docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id" + done | sort + echo "\$ curl LB stats: backend api, server status check" + curl -fsS "$stats" | python3 -c 'import csv,sys +for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): + if r["pxname"] == "api" and r["status"] != "MAINT (resolution)": + print(r["svname"], r["status"], r["check_status"], r["check_code"])' + echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id" + for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done +} > evidence/stack.txt +{ + header + echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)" + echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes" + echo "compose: $(docker compose version --short)" + echo "images:" + grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /' + echo "python in image: $(docker compose exec -T api python -V)" + echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)" + echo "packages (uv.lock):" + for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do + printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)" + done + echo "process users (service uid command):" + for id in $(docker compose ps -q); do + service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id") + docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}' + done | sort -u +} > evidence/environment.txt +cat evidence/stack.txt evidence/environment.txt diff --git a/scripts/evidence-storage.sh b/scripts/evidence-storage.sh index 0bd2d2d1..21269afc 100755 --- a/scripts/evidence-storage.sh +++ b/scripts/evidence-storage.sh @@ -3,12 +3,12 @@ set -eu cd "$(dirname "$0")/.." psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; } header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; } -counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM schema_migrations) versions" +counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM schema_migrations) versions" kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1" tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1" { header - for table in users sessions chats chat_members messages schema_migrations; do + for table in users sessions chats chat_members messages idempotency_keys schema_migrations; do echo "\$ psql -c '\\d+ $table'" psql "\\d+ $table" done @@ -38,7 +38,7 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR echo "(пусто: секретов, .env, дампов и volumes в Git нет)" echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password .env backups/" git check-ignore -v secrets/database_url secrets/postgres_password .env backups/ - echo "\$ docker volume ls --filter name=mtusi2026lab2" - docker volume ls --filter name=mtusi2026lab2 + echo "\$ docker volume ls --filter name=mtusi2026lab3" + docker volume ls --filter name=mtusi2026lab3 } > evidence/secrets.txt echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены" diff --git a/scripts/failover-evidence.sh b/scripts/failover-evidence.sh new file mode 100755 index 00000000..1bd18328 --- /dev/null +++ b/scripts/failover-evidence.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +replicas() { + docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}") +} +tmp=evidence/failover.tmp +{ + echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" + echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt" + replicas + echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh" + make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}" + echo "== after: api replicas" + replicas +} > "$tmp" 2>&1 +stopped=$(replicas | awk '$2 == "exited" {print $1}') +{ + echo "== stopped replica: $stopped; last log lines of its process" + docker logs --tail 6 "$stopped" 2>&1 + echo "\$ scripts/restore.sh" + scripts/restore.sh + echo "== after restore" + replicas +} >> "$tmp" 2>&1 +mv "$tmp" "evidence/failover-$stopped.txt" +cat "evidence/failover-$stopped.txt" diff --git a/scripts/persistence-evidence.sh b/scripts/persistence-evidence.sh index c7ba6877..f10ce0c8 100755 --- a/scripts/persistence-evidence.sh +++ b/scripts/persistence-evidence.sh @@ -2,16 +2,13 @@ set -eu cd "$(dirname "$0")/.." started() { - for service in $(docker compose config --services); do - id=$(docker compose ps -aq "$service") - if [ -n "$id" ]; then - printf '%-10s %s\n' "$service" "$(docker inspect --format '{{.State.StartedAt}} {{.State.Status}}' "$id")" - fi - done + for id in $(docker compose ps -aq); do + docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id" + done | sort } echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)" -echo "== before: service StartedAt status" +echo "== before: service hostname StartedAt status" started make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}" -echo "== after: service StartedAt status" +echo "== after: service hostname StartedAt status" started