Script the overload leftovers check with raw output

overload-leftovers.txt was assembled by hand with summarised reconciler
reports. make overload-leftovers now runs the dry-run, the status
counts, the wait for the sweep after the grace period, the audit and
swept counts of the replicas and the dry-run afterwards, and writes
their raw output.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
Artyom
2026-10-30 09:17:15 +03:00
co-authored by Claude Opus 5.5
parent 61dcb6bf13
commit 6ffb90d1d1
3 changed files with 45 additions and 1 deletions
+1
View File
@@ -30,6 +30,7 @@ make s3-check # приватный bucket вне API, огра
make upload-faults # «4»: негативный suite через LB, компенсация, уборка после grace, обрыв и зависание тела
make upload-memory ARGS='--out evidence/upload-memory/run1' # «4»: пик памяти реплик и LB при 10/20/40 загрузках 1 и 10 MiB
make load-evidence # tests/load.py ×3 клиентом-контейнером в сети edge → evidence/load.txt, evidence/load/
make overload-leftovers # после upload-memory: намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt
make clean-clone-evidence # блок «Запуск из чистого clone» REPORT.md в свежем клоне рядом → evidence/clean-clone.txt
make reconcile ARGS="--apply" # трек 2: сверка PostgreSQL ↔ S3 (без ARGS — dry-run), one-off сервис reconciler
make reconcile-demo # трек 2: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/
+5 -1
View File
@@ -22,7 +22,7 @@ PYTEST_ARGS ?=
persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \
limits detection failover-each rolling-update level4 \
scenario s3-check attachments-evidence upload-faults upload-memory \
reconcile reconcile-demo reconcile-cost load-evidence clean-clone-evidence
reconcile reconcile-demo reconcile-cost load-evidence clean-clone-evidence overload-leftovers
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -35,6 +35,7 @@ help:
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
@echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB"
@echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/"
@echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt"
@echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt"
@echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:"
@echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/"
@@ -157,6 +158,9 @@ upload-memory:
load-evidence:
sh scripts/load-evidence.sh
overload-leftovers:
sh scripts/overload-leftovers.sh
clean-clone-evidence:
sh scripts/clean-clone.sh
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
limit="${WAIT_SECONDS:-1500}"
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
since=$(date -u +%FT%TZ)
{
echo "date: $since commit: $(git rev-parse --short HEAD)"
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
echo "\$ make reconcile # dry-run"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
started=$(date +%s)
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
echo "pending = $(pending) в $(date -u +%FT%TZ)"
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
reasons, swept, times = collections.Counter(), 0, []
for line in sys.stdin:
if not line.startswith("{"):
continue
event = json.loads(line)
if event.get("event") == "audit.attachment_discarded":
reasons[event["reason"]] += 1
times.append(event["ts"][:19] + "Z")
elif event.get("event") == "attachments.swept":
swept += event["count"]
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
echo "\$ psql -c \"$statuses\""
sql -c "$statuses"
echo "\$ make reconcile # dry-run после уборки"
docker compose run --rm --no-deps -T reconciler 2>/dev/null
} > evidence/overload-leftovers.txt
cat evidence/overload-leftovers.txt