Record which replica stops and which services restart
scripts/failover-evidence.sh lists the api containers by hostname, the X-Instance-Id they answer with, before and after make test-failover, names the container left exited, keeps its last log lines with the gunicorn graceful shutdown, then restores it and lists the replicas again. scripts/evidence-stack.sh records the compose config check, published ports, the missing port of a replica, hostnames, networks, limits, process users, the balancer view of every server and the alternating X-Instance-Id through the balancer, plus the environment and image versions. The persistence and storage evidence scripts now list every replica and the idempotency key table. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ceb266ed05
commit
ae8fb0a48e
@@ -11,7 +11,7 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover \
|
||||
install lint format typecheck student-test \
|
||||
secrets up down reset-data migrate ps logs restore lb-check \
|
||||
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
|
||||
persistence-evidence storage-evidence cache-evidence test-faults seed explain
|
||||
|
||||
help:
|
||||
@@ -86,6 +86,12 @@ restore:
|
||||
lb-check:
|
||||
$(PYTHON) scripts/probe.py check --out evidence/lb-check.json
|
||||
|
||||
failover-evidence:
|
||||
sh scripts/failover-evidence.sh "$(BASE_URL)"
|
||||
|
||||
stack-evidence:
|
||||
sh scripts/evidence-stack.sh
|
||||
|
||||
persistence-evidence:
|
||||
sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt
|
||||
cat evidence/persistence.txt
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
|
||||
- [x] 6.1 `scripts/stop-one.sh`, `scripts/restart.sh` (вместо версии лабы 2), `scripts/restore.sh`, `scripts/lb-wait.sh`
|
||||
- [x] 6.2 `scripts/probe.py` (`check`), ruff для `scripts/*.py`
|
||||
- [ ] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен
|
||||
- [x] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен
|
||||
- [ ] 6.4 Проверка группы: `make test`, `make lb-check`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `scripts/restore.sh` между ними, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`
|
||||
|
||||
## 7. Отчёт
|
||||
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
base=${1:-http://$(docker compose port lb 8080)}
|
||||
stats=http://$(docker compose port lb 8404)/stats\;csv
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
{
|
||||
header
|
||||
echo "\$ docker compose --env-file .env config --quiet && echo valid"
|
||||
docker compose --env-file .env config --quiet && echo valid
|
||||
echo "\$ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'"
|
||||
docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}'
|
||||
echo "\$ docker compose config | grep -c container_name"
|
||||
docker compose config | grep -c container_name || true
|
||||
echo "\$ docker compose port api 8080 # реплики не публикуют порт"
|
||||
docker compose port api 8080 2>&1 || true
|
||||
echo "\$ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти"
|
||||
for id in $(docker compose ps -q); do
|
||||
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.Name}} nets={{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}} user={{.Config.User}} ro={{.HostConfig.ReadonlyRootfs}}' "$id"
|
||||
done | sort
|
||||
echo "\$ curl LB stats: backend api, server status check"
|
||||
curl -fsS "$stats" | python3 -c 'import csv,sys
|
||||
for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
if r["pxname"] == "api" and r["status"] != "MAINT (resolution)":
|
||||
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
|
||||
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
|
||||
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
|
||||
} > evidence/stack.txt
|
||||
{
|
||||
header
|
||||
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
|
||||
echo "docker: $(docker version -f '{{.Server.Version}}'), VM: $(docker info -f '{{.NCPU}}') CPU, $(docker info -f '{{.MemTotal}}') bytes"
|
||||
echo "compose: $(docker compose version --short)"
|
||||
echo "images:"
|
||||
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
|
||||
echo "python in image: $(docker compose exec -T api python -V)"
|
||||
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
|
||||
echo "packages (uv.lock):"
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do
|
||||
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
|
||||
done
|
||||
echo "process users (service uid command):"
|
||||
for id in $(docker compose ps -q); do
|
||||
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
|
||||
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
|
||||
done | sort -u
|
||||
} > evidence/environment.txt
|
||||
cat evidence/stack.txt evidence/environment.txt
|
||||
@@ -3,12 +3,12 @@ set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM schema_migrations) versions"
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM schema_migrations) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
|
||||
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
|
||||
{
|
||||
header
|
||||
for table in users sessions chats chat_members messages schema_migrations; do
|
||||
for table in users sessions chats chat_members messages idempotency_keys schema_migrations; do
|
||||
echo "\$ psql -c '\\d+ $table'"
|
||||
psql "\\d+ $table"
|
||||
done
|
||||
@@ -38,7 +38,7 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
|
||||
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
|
||||
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password .env backups/"
|
||||
git check-ignore -v secrets/database_url secrets/postgres_password .env backups/
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab2"
|
||||
docker volume ls --filter name=mtusi2026lab2
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab3"
|
||||
docker volume ls --filter name=mtusi2026lab3
|
||||
} > evidence/secrets.txt
|
||||
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
replicas() {
|
||||
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
|
||||
}
|
||||
tmp=evidence/failover.tmp
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
|
||||
replicas
|
||||
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
echo "== after: api replicas"
|
||||
replicas
|
||||
} > "$tmp" 2>&1
|
||||
stopped=$(replicas | awk '$2 == "exited" {print $1}')
|
||||
{
|
||||
echo "== stopped replica: $stopped; last log lines of its process"
|
||||
docker logs --tail 6 "$stopped" 2>&1
|
||||
echo "\$ scripts/restore.sh"
|
||||
scripts/restore.sh
|
||||
echo "== after restore"
|
||||
replicas
|
||||
} >> "$tmp" 2>&1
|
||||
mv "$tmp" "evidence/failover-$stopped.txt"
|
||||
cat "evidence/failover-$stopped.txt"
|
||||
@@ -2,16 +2,13 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
started() {
|
||||
for service in $(docker compose config --services); do
|
||||
id=$(docker compose ps -aq "$service")
|
||||
if [ -n "$id" ]; then
|
||||
printf '%-10s %s\n' "$service" "$(docker inspect --format '{{.State.StartedAt}} {{.State.Status}}' "$id")"
|
||||
fi
|
||||
done
|
||||
for id in $(docker compose ps -aq); do
|
||||
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
|
||||
done | sort
|
||||
}
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "== before: service StartedAt status"
|
||||
echo "== before: service hostname StartedAt status"
|
||||
started
|
||||
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
echo "== after: service StartedAt status"
|
||||
echo "== after: service hostname StartedAt status"
|
||||
started
|
||||
|
||||
Reference in New Issue
Block a user