From 2fa1b45da2caf58f3f3439dcd4e23b68f772d2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 22:49:40 +0300 Subject: [PATCH] test(pipeline): isolate durable stream snapshot from concurrent relay --- .github/workflows/application.yml | 27 +++++++ SCHEMA.md | 48 ++++++++++++ evidence/pipeline-first-failed.json | 109 ++++++++++++++++++++++++++++ evidence/pipeline-first-failure.txt | 12 +++ infra/README.md | 17 ++++- scripts/stop-worker.sh | 3 + scripts/verify_pipeline.py | 4 +- 7 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 SCHEMA.md create mode 100644 evidence/pipeline-first-failed.json create mode 100644 evidence/pipeline-first-failure.txt create mode 100755 scripts/stop-worker.sh diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..cb469576 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,27 @@ +name: Application integration +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + COMPOSE_FILE: compose.yaml:compose.test.yaml + NO_PROXY: localhost,127.0.0.1 + no_proxy: localhost,127.0.0.1 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + version: 0.13.7 + - run: uv sync --locked + - run: make bootstrap check validate lint unit PYTHON=.venv/bin/python + - run: VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait + - run: make test test-student PYTHON=.venv/bin/python BASE_URL=http://127.0.0.1:8085 + - run: uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-ci-seed.json + - run: uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-ci.json + - run: uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-ci-seed.json --output evidence/gc-ci.json + - run: uv run --locked python -m scripts.verify_cluster --output evidence/cluster-ci.json + - if: always() + run: docker compose down --volumes --remove-orphans diff --git a/SCHEMA.md b/SCHEMA.md new file mode 100644 index 00000000..62848eae --- /dev/null +++ b/SCHEMA.md @@ -0,0 +1,48 @@ +# PostgreSQL / S3 / broker — ЛР5 + +```mermaid +flowchart LR + C[Client] --> LB[HAProxy] + LB --> A[API 1/2] + A --> S[Private S3 staging] + A --> P[(PG attachment + job + outbox)] + P --> R[Relay + watchdog] + R --> B[(Valkey Streams + AOF)] + B --> W[Worker 2 bounded child slots] + W --> V[S3 versioned original + 3 variants] + W --> E[(PG fenced atomic publication)] + E --> A +``` + +Core: users→sessions; chats↔members; chats→messages; message_attachments +имеет составные FK chat/message и chat/attachment. Сессии и idempotency +сохраняются в PG; cache не является источником истины. Row locks и unique +(user,method,path,key) сохраняют Message+result атомарно. + +Attachment хранит свойства original, target pipeline_version и +published_version. Jobs unique(attachment_id,pipeline_version), UUID logical ID, +state,attempt/cycle,fence,lease/owner,next_at,safeerror. Outbox содержит только +schema-valid envelope, due/published. Consumer group хранит pending entries, +но окончательные решения находятся в PG. Failed registry — jobs.state=failed. + +Результаты: attachment_variants PK(attachment_id,pipeline_version,name), +unique object_key; effect_publications PK(attachment_id,pipeline_version). +Worker в одной транзакции проверяет target version, fence, live lease и +processing, записывает4variants+effect_publication+ready. Object bytes +предшествуют COMMIT; дубли могут повторить PUT тех же bytes/keys. +Публикацию новой версии нельзя перезаписать старым fence/version. + +Upload reservation120s предотвращает поздний COMMIT после GC: row lock плюс +проверка expiry нужны и при handoff, и при удалении просроченного reservation. +Unconfirmed bytes не становятся accepted jobs сами по себе. + +S3: staging `objects/staging//original`, final +`objects//v/{original,thumbnail,crop,optimized}`. +Filename — только отображаемое поле, не путь. IAM ограничен messenger/objects/*, +корневые credentials получает только S3/bootstrap. PG содержит metadata/SHA, +а не bytes. Broker envelope содержит ID/version/attempt/time, не storage paths. + +pipeline_audit сохраняет explicit operator replay/version; pipeline_events — +наблюдаемые решения без secrets. pipeline_faults — test-only single-use hooks, +выключены без PIPELINE_FAULTS=1. В production API не имеет broker credentials; +test controller появляется только в compose.test.yaml и не публикует портов. diff --git a/evidence/pipeline-first-failed.json b/evidence/pipeline-first-failed.json new file mode 100644 index 00000000..5550214c --- /dev/null +++ b/evidence/pipeline-first-failed.json @@ -0,0 +1,109 @@ +{ + "source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2", + "date": "2026-11-15T19:46:36.154413+00:00", + "track": 2, + "complete": false, + "cases": [ + { + "case": "staging_before_handoff_crash", + "no_202": true, + "no_attachment_or_job_created": true, + "orphan_retained_for_cleanup": true + }, + { + "case": "commit_before_publish_and_worker_stopped", + "attachment": "f554e320-4881-4087-b813-9f28607fc3b1", + "effect": { + "job": { + "id": "1016e737-7ec1-4fde-9946-a342d70b2a71", + "attachment_id": "f554e320-4881-4087-b813-9f28607fc3b1", + "pipeline_version": 1, + "source_key": "objects/staging/f554e320-4881-4087-b813-9f28607fc3b1/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:47:10.665115+00:00", + "error_code": null, + "created_at": "2026-11-15 19:47:10.665115+00:00", + "updated_at": "2026-11-15 19:47:36.689673+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop", + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized", + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original", + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:47:34.529681+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772054509-0", + "outbox": 11 + }, + "created_at": "2026-11-15 19:47:34.584052+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:47:36.690043+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2740797910000765, + "cpu_seconds": 0.2354738329999997, + "rss_peak_mib": 79.3125 + }, + "created_at": "2026-11-15 19:47:36.847178+00:00" + } + ] + }, + "queued_link_allowed": true, + "api_and_broker_restarted": true + } + ] +} diff --git a/evidence/pipeline-first-failure.txt b/evidence/pipeline-first-failure.txt new file mode 100644 index 00000000..eea202b5 --- /dev/null +++ b/evidence/pipeline-first-failure.txt @@ -0,0 +1,12 @@ +PASS: staging_before_handoff_crash +PASS: commit_before_publish_and_worker_stopped +Traceback (most recent call last): + File "", line 203, in _run_module_as_main + File "", line 88, in _run_code + File "/scripts/verify_pipeline.py", line 602, in + main() + ~~~~^^ + File "/scripts/verify_pipeline.py", line 318, in main + assert restored == ids + ^^^^^^^^^^^^^^^ +AssertionError diff --git a/infra/README.md b/infra/README.md index 5a39e862..c1306536 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,16 @@ -# Инфраструктура студента +# Инфраструктура ЛР5 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология и реальные pinned versions — compose.yaml; сборка MinIO/mc из +закреплённых commits — s3.Dockerfile; bucket/IAM — s3-bootstrap.sh и s3-policy.json. +Credentials создаёт scripts/bootstrap.py, значения не входят в Git/image. +API/worker/reconciler получают scoped application keys, relay только PG+broker, +root keys доступны только S3/bootstrap. Консоль S3 слушает loopback контейнера. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +Сервисы: postgres persistent volume; cache disposable; s3 persistent volume; +broker отдельный Valkey Streams persistent AOF volume; relay/outbox/watchdog; +worker отдельный2CPU1536MiB2slots; api2replicas; lb один localhost:8085. +S3/broker/PG/API host ports не публикуются. Сеть — private Compose default; +production TLS/изоляция от потери host не заявлены. Детали — REPORT.md/SCHEMA.md. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +compose.test.yaml включает fault instrumentation и отдельный controller, +без публичных endpoints; обычный запуск не имеет этого controller/hooks. diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..2f4ef642 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +docker compose stop --timeout 1 worker diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index 78281d38..6f850068 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -310,12 +310,14 @@ def main(): f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))" ) ) + stop("relay") ids = inside("print(json.dumps(queue.xrange(STREAM)))") compose("kill", "-s", "SIGKILL", "broker") start("broker") wait(lambda: inside("print(json.dumps(queue.ping()))")) restored = inside("print(json.dumps(queue.xrange(STREAM)))") - assert restored == ids + assert restored == ids, {"before": ids, "after": restored} + start("relay") start("worker") save( "confirmed_broker_entry_survives_sigkill",