Record which replica stops and which services restart

scripts/failover-evidence.sh lists the api containers by hostname, the
X-Instance-Id they answer with, before and after make test-failover,
names the container left exited, keeps its last log lines with the
gunicorn graceful shutdown, then restores it and lists the replicas
again. scripts/evidence-stack.sh records the compose config check,
published ports, the missing port of a replica, hostnames, networks,
limits, process users, the balancer view of every server and the
alternating X-Instance-Id through the balancer, plus the environment
and image versions. The persistence and storage evidence scripts now
list every replica and the idempotency key table.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-28 11:44:03 +03:00
co-authored by Claude Opus 5.5
parent ceb266ed05
commit ae8fb0a48e
6 changed files with 92 additions and 14 deletions
+7 -1
View File
@@ -11,7 +11,7 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
secrets up down reset-data migrate ps logs restore lb-check \
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
persistence-evidence storage-evidence cache-evidence test-faults seed explain
help:
@@ -86,6 +86,12 @@ restore:
lb-check:
$(PYTHON) scripts/probe.py check --out evidence/lb-check.json
failover-evidence:
sh scripts/failover-evidence.sh "$(BASE_URL)"
stack-evidence:
sh scripts/evidence-stack.sh
persistence-evidence:
sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt
cat evidence/persistence.txt
@@ -33,7 +33,7 @@
- [x] 6.1 `scripts/stop-one.sh`, `scripts/restart.sh` (вместо версии лабы 2), `scripts/restore.sh`, `scripts/lb-wait.sh`
- [x] 6.2 `scripts/probe.py` (`check`), ruff для `scripts/*.py`
- [ ] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен
- [x] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен
- [ ] 6.4 Проверка группы: `make test`, `make lb-check`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `scripts/restore.sh` между ними, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`
## 7. Отчёт
+48
View File
@@ -0,0 +1,48 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
base=${1:-http://$(docker compose port lb 8080)}
stats=http://$(docker compose port lb 8404)/stats\;csv
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
{
header
echo "\$ docker compose --env-file .env config --quiet && echo valid"
docker compose --env-file .env config --quiet && echo valid
echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'"
docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'
echo "\$ docker compose config | grep -c container_name"
docker compose config | grep -c container_name || true
echo "\$ docker compose port api 8080 # реплики не публикуют порт"
docker compose port api 8080 2>&1 || true
echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти"
for id in $(docker compose ps -q); do
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id"
done | sort
echo "\$ curl LB stats: backend api, server status check"
curl -fsS "$stats" | python3 -c 'import csv,sys
for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
if r["pxname"] == "api" and r["status"] != "MAINT (resolution)":
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
} > evidence/stack.txt
{
header
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes"
echo "compose: $(docker compose version --short)"
echo "images:"
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
echo "python in image: $(docker compose exec -T api python -V)"
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
echo "process users (service uid command):"
for id in $(docker compose ps -q); do
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
done | sort -u
} > evidence/environment.txt
cat evidence/stack.txt evidence/environment.txt
+4 -4
View File
@@ -3,12 +3,12 @@ set -eu
cd "$(dirname "$0")/.."
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM schema_migrations) versions"
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM schema_migrations) versions"
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
{
header
for table in users sessions chats chat_members messages schema_migrations; do
for table in users sessions chats chat_members messages idempotency_keys schema_migrations; do
echo "\$ psql -c '\\d+ $table'"
psql "\\d+ $table"
done
@@ -38,7 +38,7 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password .env backups/"
git check-ignore -v secrets/database_url secrets/postgres_password .env backups/
echo "\$ docker volume ls --filter name=mtusi2026lab2"
docker volume ls --filter name=mtusi2026lab2
echo "\$ docker volume ls --filter name=mtusi2026lab3"
docker volume ls --filter name=mtusi2026lab3
} > evidence/secrets.txt
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replicas() {
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
}
tmp=evidence/failover.tmp
{
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
replicas
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
echo "== after: api replicas"
replicas
} > "$tmp" 2>&1
stopped=$(replicas | awk '$2 == "exited" {print $1}')
{
echo "== stopped replica: $stopped; last log lines of its process"
docker logs --tail 6 "$stopped" 2>&1
echo "\$ scripts/restore.sh"
scripts/restore.sh
echo "== after restore"
replicas
} >> "$tmp" 2>&1
mv "$tmp" "evidence/failover-$stopped.txt"
cat "evidence/failover-$stopped.txt"
+5 -8
View File
@@ -2,16 +2,13 @@
set -eu
cd "$(dirname "$0")/.."
started() {
for service in $(docker compose config --services); do
id=$(docker compose ps -aq "$service")
if [ -n "$id" ]; then
printf '%-10s %s\n' "$service" "$(docker inspect --format '{{.State.StartedAt}} {{.State.Status}}' "$id")"
fi
done
for id in $(docker compose ps -aq); do
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
done | sort
}
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
echo "== before: service StartedAt status"
echo "== before: service hostname StartedAt status"
started
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
echo "== after: service StartedAt status"
echo "== after: service hostname StartedAt status"
started