Script the overload leftovers check with raw output
overload-leftovers.txt was assembled by hand with summarised reconciler reports. make overload-leftovers now runs the dry-run, the status counts, the wait for the sweep after the grace period, the audit and swept counts of the replicas and the dry-run afterwards, and writes their raw output. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
61dcb6bf13
commit
6ffb90d1d1
@@ -30,6 +30,7 @@ make s3-check # приватный bucket вне API, огра
|
||||
make upload-faults # «4»: негативный suite через LB, компенсация, уборка после grace, обрыв и зависание тела
|
||||
make upload-memory ARGS='--out evidence/upload-memory/run1' # «4»: пик памяти реплик и LB при 10/20/40 загрузках 1 и 10 MiB
|
||||
make load-evidence # tests/load.py ×3 клиентом-контейнером в сети edge → evidence/load.txt, evidence/load/
|
||||
make overload-leftovers # после upload-memory: намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt
|
||||
make clean-clone-evidence # блок «Запуск из чистого clone» REPORT.md в свежем клоне рядом → evidence/clean-clone.txt
|
||||
make reconcile ARGS="--apply" # трек 2: сверка PostgreSQL ↔ S3 (без ARGS — dry-run), one-off сервис reconciler
|
||||
make reconcile-demo # трек 2: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/
|
||||
|
||||
@@ -22,7 +22,7 @@ PYTEST_ARGS ?=
|
||||
persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \
|
||||
limits detection failover-each rolling-update level4 \
|
||||
scenario s3-check attachments-evidence upload-faults upload-memory \
|
||||
reconcile reconcile-demo reconcile-cost load-evidence clean-clone-evidence
|
||||
reconcile reconcile-demo reconcile-cost load-evidence clean-clone-evidence overload-leftovers
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -35,6 +35,7 @@ help:
|
||||
@echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда"
|
||||
@echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB"
|
||||
@echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/"
|
||||
@echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt"
|
||||
@echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt"
|
||||
@echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:"
|
||||
@echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/"
|
||||
@@ -157,6 +158,9 @@ upload-memory:
|
||||
load-evidence:
|
||||
sh scripts/load-evidence.sh
|
||||
|
||||
overload-leftovers:
|
||||
sh scripts/overload-leftovers.sh
|
||||
|
||||
clean-clone-evidence:
|
||||
sh scripts/clean-clone.sh
|
||||
|
||||
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
limit="${WAIT_SECONDS:-1500}"
|
||||
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
|
||||
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
|
||||
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $since commit: $(git rev-parse --short HEAD)"
|
||||
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
|
||||
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
|
||||
echo "\$ make reconcile # dry-run"
|
||||
docker compose run --rm --no-deps -T reconciler 2>/dev/null
|
||||
echo "\$ psql -c \"$statuses\""
|
||||
sql -c "$statuses"
|
||||
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
|
||||
started=$(date +%s)
|
||||
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
|
||||
echo "pending = $(pending) в $(date -u +%FT%TZ)"
|
||||
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
|
||||
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
|
||||
reasons, swept, times = collections.Counter(), 0, []
|
||||
for line in sys.stdin:
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
event = json.loads(line)
|
||||
if event.get("event") == "audit.attachment_discarded":
|
||||
reasons[event["reason"]] += 1
|
||||
times.append(event["ts"][:19] + "Z")
|
||||
elif event.get("event") == "attachments.swept":
|
||||
swept += event["count"]
|
||||
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
|
||||
echo "\$ psql -c \"$statuses\""
|
||||
sql -c "$statuses"
|
||||
echo "\$ make reconcile # dry-run после уборки"
|
||||
docker compose run --rm --no-deps -T reconciler 2>/dev/null
|
||||
} > evidence/overload-leftovers.txt
|
||||
cat evidence/overload-leftovers.txt
|
||||
Reference in New Issue
Block a user