diff --git a/AGENTS.md b/AGENTS.md index f2a43363..31975fb2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,7 @@ make s3-check # приватный bucket вне API, огра make upload-faults # «4»: негативный suite через LB, компенсация, уборка после grace, обрыв и зависание тела make upload-memory ARGS='--out evidence/upload-memory/run1' # «4»: пик памяти реплик и LB при 10/20/40 загрузках 1 и 10 MiB make load-evidence # tests/load.py ×3 клиентом-контейнером в сети edge → evidence/load.txt, evidence/load/ +make overload-leftovers # после upload-memory: намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt make clean-clone-evidence # блок «Запуск из чистого clone» REPORT.md в свежем клоне рядом → evidence/clean-clone.txt make reconcile ARGS="--apply" # трек 2: сверка PostgreSQL ↔ S3 (без ARGS — dry-run), one-off сервис reconciler make reconcile-demo # трек 2: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/ diff --git a/Makefile b/Makefile index 3f346b7b..1def346b 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ PYTEST_ARGS ?= persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \ limits detection failover-each rolling-update level4 \ scenario s3-check attachments-evidence upload-faults upload-memory \ - reconcile reconcile-demo reconcile-cost load-evidence clean-clone-evidence + reconcile reconcile-demo reconcile-cost load-evidence clean-clone-evidence overload-leftovers help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -35,6 +35,7 @@ help: @echo "upload-faults: негативный suite, компенсация, уборка, обрыв и зависание загрузки против стенда" @echo "upload-memory ARGS='--out evidence/upload-memory/runN': пик памяти реплик при 10/20/40 загрузках 1 и 10 MiB" @echo "load-evidence: tests/load.py ×3 клиентом-контейнером в сети edge стенда → evidence/load.txt, evidence/load/" + @echo "overload-leftovers: после upload-memory — намерения от 503, сверка и уборка после grace → evidence/overload-leftovers.txt" @echo "clean-clone-evidence: блок «Запуск из чистого clone» в свежем клоне рядом → evidence/clean-clone.txt" @echo "reconcile ARGS='--apply --deep': сверка PostgreSQL ↔ S3 (без ARGS — dry-run); трек 2:" @echo "reconcile-demo: внедрение ошибок, dry-run/apply/повтор, безопасность, нагрузка → evidence/reconcile/" @@ -157,6 +158,9 @@ upload-memory: load-evidence: sh scripts/load-evidence.sh +overload-leftovers: + sh scripts/overload-leftovers.sh + clean-clone-evidence: sh scripts/clean-clone.sh diff --git a/scripts/overload-leftovers.sh b/scripts/overload-leftovers.sh new file mode 100755 index 00000000..fe798fb7 --- /dev/null +++ b/scripts/overload-leftovers.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +limit="${WAIT_SECONDS:-1500}" +sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; } +pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; } +statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status" +since=$(date -u +%FT%TZ) +{ + echo "date: $since commit: $(git rev-parse --short HEAD)" + echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по" + echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же" + echo "\$ make reconcile # dry-run" + docker compose run --rm --no-deps -T reconciler 2>/dev/null + echo "\$ psql -c \"$statuses\"" + sql -c "$statuses" + echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done" + started=$(date +%s) + until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done + echo "pending = $(pending) в $(date -u +%FT%TZ)" + echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept" + docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys +reasons, swept, times = collections.Counter(), 0, [] +for line in sys.stdin: + if not line.startswith("{"): + continue + event = json.loads(line) + if event.get("event") == "audit.attachment_discarded": + reasons[event["reason"]] += 1 + times.append(event["ts"][:19] + "Z") + elif event.get("event") == "attachments.swept": + swept += event["count"] +print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))' + echo "\$ psql -c \"$statuses\"" + sql -c "$statuses" + echo "\$ make reconcile # dry-run после уборки" + docker compose run --rm --no-deps -T reconciler 2>/dev/null +} > evidence/overload-leftovers.txt +cat evidence/overload-leftovers.txt