diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a..0d79a30 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,39 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Илья Тюнин, БВТ2303 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: трек 2 «Доказанная идемпотентность pipeline» -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: upload переведён на 202 `queued`; байты +принимаются в S3 `staging/*`, задание и outbox коммитятся в PostgreSQL одной +транзакцией. Добавлены RabbitMQ 4.1 (quorum queue, DLX/DLQ), отдельный worker +с lease/heartbeat, retry с equal jitter, идемпотентными вариантами и DLQ, +операторский CLI (`status`, `failed`, `replay`, `reprocess`, `queues`, +`publish-raw`), версии pipeline v1/v2 и fault injection через Redis. Из lab4 +сохранены API, auth, чаты, сообщения, direct upload, кэш, drain, sweeper и +resilience hooks. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Команды запуска из чистого clone: `make init-env`, `make stand-up`, +`make wait-ready`, `make test`, `make test-e2e`, `make test-dependencies`, +`make lab5-demo`; отказы трека 2 — стенд с +`FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000` +и `make lab5-faults` / `make lab5-track2`. + +Результаты проверок и ссылка на REPORT.md: `REPORT.md`, +`evidence/lab5/summary.md`; smoke, e2e, dependency, persistence/failover, +worker-down demo и fault-сценарии. + +Ограничения / известные ошибки: single-node PostgreSQL/RabbitMQ/MinIO; +fault points только внутри процесса; S3-запись может повториться, но остаётся +побайтно детерминированной; track 1 не заявлен. + +Источники и использование ИИ: RabbitMQ reliability/confirms, amqplib 2.2.1, +sharp 0.35.5, контракты репозитория; ИИ использован для черновика плана, +ревью и редактирования текста, все проверки воспроизведены локально. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Makefile b/Makefile index 76c20a9..beb83f7 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-dependencies stand-up stand-down stand-status wait-ready init-env test-e2e s3-security consistency bench +.PHONY: help check validate test load test-persistence test-failover test-dependencies stand-up stand-down stand-status wait-ready init-env test-e2e s3-security consistency bench lab5-demo lab5-faults lab5-track2 jobs help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes; init-env: .env + secrets; test-e2e: app e2e; s3-security; consistency; bench" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes; init-env: .env + secrets; test-e2e: app e2e; s3-security; consistency; bench; lab5-demo; lab5-faults; lab5-track2; jobs ARGS=..." check: $(PYTHON) scripts/check_contract.py @@ -67,3 +67,15 @@ consistency: bench: scripts/lab4/run_matrix.sh + +lab5-demo: + $(PYTHON) scripts/lab5/worker_down_demo.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +lab5-faults: + $(PYTHON) scripts/lab5/fault_suite.py --base-url "$(BASE_URL)" --scenario level4 --output evidence/lab5/faults-level4.json $(TLS_ARGS) + +lab5-track2: + $(PYTHON) scripts/lab5/fault_suite.py --base-url "$(BASE_URL)" --scenario track2 --repeat 3 --output evidence/lab5/track2.json $(TLS_ARGS) + +jobs: + docker compose --env-file $(COMPOSE_ENV_FILE) exec -T api node dist/jobs/jobs.cli.js $(ARGS) diff --git a/REPORT.md b/REPORT.md index e98f75e..d6b52a8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,312 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Илья Тюнин, БВТ2303 +- Целевая оценка: 5, трек 2 «Доказанная идемпотентность pipeline» +- Commit/PR: ветка `lab5`, коммиты `lab5: ...`; PR — после публикации +- Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [RabbitMQ confirms](https://www.rabbitmq.com/docs/confirms), `contracts/jobs.schema.json`, `contracts/attachment-states.md`, amqplib 2.2.1, sharp 0.35.5. ИИ использован для черновика плана, ревью кода и текста отчёта; решения и проверки выполнены и воспроизведены локально. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker 29.x с Compose v2, Python 3.9+, Node 25.7.0 для локальных +юнит/e2e-прогонов. Образы приложения собираются из этого репозитория. + +```sh +make init-env # .env (если нет) + secrets/*; значения не печатаются +make check # offline-структура и OpenAPI +make stand-up # build + PostgreSQL, Redis, MinIO, RabbitMQ, API x2, worker, HAProxy +make wait-ready # /health/ready +make test # публичный smoke: 202 -> ready -> thumbnail 256x160 +make test-e2e # e2e приложения против стенда +make test-dependencies # отказы Redis/PostgreSQL/S3, пауза S3, saturation +make test-persistence ACTION_SCRIPT=scripts/restart.sh +scripts/start-replica.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make lab5-demo # worker остановлен -> upload 202 -> рестарт broker/API -> ready +``` + +Проверка трека 2 требует включённого fault injection и укороченных таймаутов: + +```sh +FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 JOB_LEASE_SECONDS=8 \ +JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 make stand-up +make lab5-faults # level4 -> evidence/lab5/faults-level4.json +make lab5-track2 # track2 x3 -> evidence/lab5/track2.json +make stand-up # вернуть штатные значения +``` + +Остановка: `make stand-down`. Намеренное удаление учебных данных (необратимо): +`docker compose --env-file .env down -v` — удаляет PostgreSQL, Redis, MinIO и +RabbitMQ volumes. Секреты лежат в `secrets/` и в Git не попадают. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```mermaid +flowchart LR + C[client] --> H[HAProxy] + H --> A1[API 1] + H --> A2[API 2] + A1 --> PG[(PostgreSQL)] + A2 --> PG + A1 --> S3S[(S3 staging/*)] + A1 --> O[(job_outbox)] + O --> R[Outbox relay] + R --> MQ[(RabbitMQ quorum queue)] + MQ --> W[worker] + W --> S3F[(S3 attachments/*)] + W --> PG + A1 --> RDS[(Redis: sessions, cache, faults)] +``` + +Источник истины — PostgreSQL. Брокер только доставляет сообщение; S3 хранит +байты. API и worker — разные процессы, worker не импортирует HTTP-модули. + +### Состояния + +| `attachments.status` | Значения | +| --- | --- | +| публичные | `queued`, `processing`, `ready`, `failed` | +| внутренние | `pending` (идёт приём), `deleting`, `rejected` (lab4) | + +| `attachment_jobs.status` | Значения | +| --- | --- | +| | `queued`, `processing`, `succeeded`, `failed`, `superseded` | + +Правила переходов выполняются в транзакции, меняющей job: + +- claim: `queued -> processing` только при `attachments.status='queued'`; +- success: `ready`, если `coalesce(ready_pipeline_version,0) <= pipeline_version`, + иначе job становится `superseded`, а строка вложения не меняется; +- retry: `processing -> queued` только при `attachments.status='processing'`; +- final failure: `failed` + `error_code`, только если вложение не `ready`; +- replay: `failed -> queued`; +- `ready` никогда не откатывается; старая job не перезаписывает новый результат. + +### Что уже сохранено в момент 202 + +До ответа 202 выполняются и фиксируются одним COMMIT: + +1. все байты приняты в `staging//original`; размер и SHA-256 сверены, + `ImageInspector` и полный decode проверили контейнер и пиксели; +2. `attachments` — строка `queued` с `size_bytes`, `sha256`, `width`, `height`; +3. `attachment_jobs` — логическая работа `(attachment_id, pipeline_version)`, + `attempt=1`, `max_attempts`; +4. `job_outbox` — намерение опубликовать сообщение. + +Только после COMMIT вызывается fault point `api.after_commit_before_publish` и +relay. Брокер не входит в `/health/ready`: outbox позволяет принимать загрузки +при недоступном RabbitMQ, а relay опубликует задание после восстановления. + +### Окна отказа + +| Окно из contract | Механизм | Сценарий | Доказательство | +| --- | --- | --- | --- | +| после staging до записи задания | строка `pending` + `expires_at`; sweeper удаляет staging и строку | lab4 sweep + `make test-dependencies` | `make-test-dependencies.txt` | +| после COMMIT до publish | transactional outbox, relay повторяет непомеченные строки | `crash_after_commit_before_publish` | `faults-level4.json` | +| после publish до confirm | ConfirmChannel; без confirm транзакция relay откатывается | `relay_crash_after_publish` | `track2.json` | +| после записи S3 до COMMIT результата | lease истекает, reaper возвращает job в `queued`; повторный рендер детерминирован | `crash_after_side_effect` | `track2.json` | +| после COMMIT результата до ack | redelivery видит `succeeded` и отвечает `duplicate_skipped` | `crash_after_commit_before_ack` | `track2.json` | + +### Publisher confirm и consumer ack + +- **Publisher confirm** — брокер принял persistent-сообщение. Outbox помечает + строку опубликованной только после confirm. +- **Consumer ack** — worker устойчиво зафиксировал эффект в PostgreSQL и S3. + Ack после commit; падение между commit и ack даёт redelivery и no-op. + +Это at-least-once доставка. Ключ эффекта — `(attachment_id, pipeline_version, +variant)` (PK `attachment_variants`); логическая работа уникальна по +`(attachment_id, pipeline_version)`. Повторная доставка не создаёт второй набор +вариантов. `attempt` увеличивается только управляемым retry/replay/истечением +lease, но не redelivery брокера. + +### Зависший worker и долгая работа + +Claim ставит `lease_owner`, `lease_token`, `lease_until`. Heartbeat продлевает +lease каждые `JOB_LEASE_SECONDS/3`; 0 обновлённых строк означает потерю lease и +аборт job. `JOB_TIMEOUT_SECONDS` — жёсткий дедлайн: `AbortController` плюс +`sharp(...).timeout({seconds})` дают transient `processing_timeout`. Reaper +возвращает job с истёкшим lease в очередь, поэтому убитый worker не оставляет +вечный `processing`. + +### DLQ, failed registry, replay + +- **DLQ** (`attachments.process.dlq`): невалидный JSON/схема и сообщение, + не совпадающее с job (`mismatch`). Это не очередь повторных попыток. +- **Failed registry**: `cli failed` — jobs в `failed` с `last_error_code`. +- **Replay** (`cli replay`) разрешён только для failed job, при отсутствии + активной job, и отказывает `already_ready`, `no_failed_job`, `job_active`, + `source_expired`. Новая попытка защищена тем же ключом эффекта; успешный + результат не исполняется повторно. + +### Staging retention + +Worker удаляет `staging//original` после успешного commit. Остатки убирает +`StagingCleaner`: для `ready` — через минуту, для `failed` — через +`STAGING_RETENTION_SECONDS` (24 ч). Pending/rejected загрузки удаляет lab4 +sweeper по `expires_at + grace`. Текущую работу cleaner не трогает. + +### Качество вариантов и метаданные + +| Вариант | v1 | v2 | +| --- | --- | --- | +| thumbnail | JPEG q80, fit 256×256, без апскейла, alpha на белом | JPEG q90 progressive mozjpeg | +| crop | JPEG q80, 128×128 cover centre (апскейл разрешён) | JPEG q90 progressive mozjpeg | +| optimized | WebP q75 effort 4, fit 1280×1280 | WebP q82 effort 5 smartSubsample | + +EXIF orientation применяется `.rotate()`; размеры берутся из +`metadata().autoOrient`. sharp по умолчанию удаляет EXIF/ICC (нет +`withMetadata`). `fitInside` использует целочисленную арифметику: 320×200 → +256×160, 1000×335 → 256×85, 10000×1 → 256×1. + +### Память worker + +База ≈250 MiB + `WORKER_CONCURRENCY × (10 MiB исходник + ≈340 MiB худший декод +40 Мп)` ≈ 950 MiB при concurrency 2. Лимит `WORKER_MEMORY=1280m`. +`big_image` загружает два `worst_decode_png` (8000×5000) параллельно: измеренный +пик — **966.3 MiB / 1.25 GiB**, `OOMKilled=false`, оба вложения дошли до ready. +Запас ~300 MiB; при росте concurrency лимит нужно поднимать вместе с бюджетом. + +### Конфигурация + +| Переменная | Фактическое значение / роль | +| --- | --- | +| `BROKER_URL_FILE` | `/run/secrets/broker_url`, amqp URL с паролем из secret | +| `BROKER_DELIVERY_LIMIT` | 10, `x-delivery-limit` quorum-очереди | +| `WORKER_CONCURRENCY` | 2, prefetch и число параллельных job | +| `JOB_MAX_ATTEMPTS` | 5, ограниченные retry | +| `JOB_TIMEOUT_SECONDS` | 60, жёсткий дедлайн рендера | +| `JOB_LEASE_SECONDS` | 30, lease + heartbeat/3 | +| `JOB_RETRY_BASE_MS` / `JOB_RETRY_MAX_MS` | 1000 / 30000, equal jitter | +| `PIPELINE_VERSION` | 1, версия для новых upload | +| `OUTBOX_POLL_INTERVAL_MS` | 500, период relay | +| `LEASE_REAPER_INTERVAL_MS` | 2000 | +| `STAGING_CLEANUP_INTERVAL_MS` | 60000, 0 отключает | +| `STAGING_RETENTION_SECONDS` | 86400 | +| `WORKER_HEALTH_PORT` | 8081 | +| `FAULT_INJECTION_ENABLED` | 0 в штатном стенде, 1 для suite | +| `WORKER_REPLICAS` / `WORKER_CPUS` / `WORKER_MEMORY` | 1 / 1.5 / 1280m | +| `BROKER_CPUS` / `BROKER_MEMORY` | 0.5 / 512m | + +Остальные переменные lab4 (`DATABASE_URL`, `REDIS_URL`, S3, лимиты загрузки, +таймауты) сохранены; `S3_ACCESS_KEY_ID_FILE`/`S3_SECRET_ACCESS_KEY_FILE` +подменяются в worker на отдельную identity. + +### Роли инфраструктуры + +| Роль | Service | Образ | Готовность | Volume | Credentials | +| --- | --- | --- | --- | --- | --- | +| API | `api` | сборка `mtusi-messenger:lab5` | `/health/ready` (PG/Redis/S3) | — | `s3_api_*`, `broker_url` | +| Worker | `worker` | та же сборка, `dist/worker.main.js` | `127.0.0.1:8081/health` | — | `s3_worker_*`, `broker_url` | +| Broker | `broker` | `rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e` | `rabbitmq-diagnostics check_port_connectivity` | `broker-data` | `rabbitmq.conf` | +| PostgreSQL | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `postgres-data` | `.env` / `DATABASE_URL` | +| Redis | `redis` | `redis:7.4.2-alpine` | `redis-cli ping` | `redis-data` | только compose-сеть | +| S3 | `s3` | `minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e` | `mc ready local` | `s3-data` | root secrets | +| S3 bootstrap | `s3-init` | `minio/mc:...@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727` | exit 0 | — | root + API/worker | +| LB | `haproxy` | `haproxy:3.2-alpine` | depends on api healthy | — | — | + +### Изменения относительно lab4 + +- upload 201 → 202 `queued`; metadata отдаёт `queued/processing/ready/failed`, + `error_code` и `variants`; +- байты пишутся в `staging/*`, worker переносит оригинал в `attachments/*`; +- добавлены `attachment_jobs`, `attachment_variants`, `job_outbox`, `job_events`; +- добавлены broker, worker, отдельные S3-identity и политики; +- `openContent` отдаёт `original/thumbnail/crop/optimized` и 409 до ready; +- сообщения можно привязывать к `queued/processing` вложениям, failed даёт 409; +- сохранены lab4-механизмы: idempotency, cache, drain, sweeper, direct upload, + failover/persistence hooks. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Сборка и lint | `npm run build && npm run lint` | без ошибок; `dist/worker.main.js` и `dist/jobs/jobs.cli.js` | `build-lint.txt` | +| Публичный smoke | `make test` | `PASS: lab5 public API scenario`; 202 → ready → thumbnail 256×160 JPEG, sha совпал | `evidence/lab5/make-test.txt` | +| «3»: API 202 только после устойчивого приёма | `make test`, `make lab5-demo` | staging + attachment + job + outbox в одном COMMIT; worker остановлен, upload принят | `make-test.txt`, `worker-down-demo.txt` | +| «3»: отдельные broker и worker | `make lab5-demo`, `docker compose ps` | broker/worker healthy; worker не в API-процессе | `compose-ps.txt`, `worker-down-demo.txt` | +| «3»: состояния и thumbnail | `make test`, `make test-e2e` | metadata `queued/processing/ready`; thumbnail 256×160; original неизменен | `make-test.txt`, `make-test-e2e.txt` | +| «3»: ограниченный retry и failed | `retry_and_replay` | 4 `retry_scheduled`, затем `failed`/`retry_exhausted`; replay → ready | `faults-level4.json` | +| «3»: принятое задание переживает рестарт broker | `make lab5-demo` | quorum-очередь сохранила сообщение после restart broker+API | `worker-down-demo.txt`, `queues.txt` | +| «4»: crop/optimized, качество, EXIF | `make test`, unit `pipelines.spec.ts` | 128×128 JPEG, WebP 320×200, EXIF 6 → 170×256, прозрачность на белом | `unit-tests.txt`, `make-test.txt` | +| «4»: outbox, crash COMMIT→publish | `crash_after_commit_before_publish` | attachment готов без ручного создания job; один `created` | `faults-level4.json` | +| «4»: backoff/jitter, permanent/transient, DLQ/failed/replay | `retry_and_replay`, `poison` | transient exhausted, DLQ +2, replay с audit | `faults-level4.json` | +| «4»: crash после S3 до ack | `crash_after_side_effect` | attempt 2, lease_expired, варианты байт-идентичны baseline | `faults-level4.json`, `track2.json` | +| «4»: timeout/lease, concurrency, очистка staging | `hung_worker`, `make lab5-demo`, CLI status | `processing_timeout` retry; `staging_present=false` после ready | `faults-level4.json`, `queues.txt` | +| «4»: память worker на 40 Мп | `big_image` | 2 × 40 Мп параллельно, пик 577–966 MiB, OOMKilled=false | `big-image-memory.txt`, `faults-level4.json` | +| «5» трек 2: версионирование, дубль, reorder, crash, stale | `make lab5-track2` | все инварианты I1–I6 истинны на 3 повторах | `track2.json` | +| «5» трек 2: отсутствие потери и перезаписи | `stale_version` | ready_pipeline_version=2, v1 `superseded`, v2 etag не изменился | `track2.json` | +| «5» трек 2: не exactly-once доставка | отчёт, `duplicate` | 5 duplicate publish дали 5 `duplicate_skipped`, S3 etag не изменился | `track2.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Гипотеза трека 2 -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +At-least-once delivery + DB-fenced claim + versioned effect keys дают +**exactly-once effects** для `(attachment_id, pipeline_version, variant)` при +дубле, reorder, crash до/после side effect и устаревшей job после новой версии. +Доставка при этом остаётся at-least-once: дубли и повторные публикации +наблюдаются и поглощаются, а не исчезают. + +### Методика + +- Стенд: 8 vCPU (i7-11370H), 15 GiB RAM; worker 1 реплика, concurrency 2, + лимит 1280 MiB; broker 0.5 CPU / 512 MiB; JOB_TIMEOUT_SECONDS=10, + JOB_LEASE_SECONDS=8, retry 500–4000 ms; `track2` повторён 3 раза. +- Перед сценариями загружается reference и запоминаются SHA-256 вариантов v1. +- Каждый сценарий начинается и заканчивается очисткой `fault:*`; timeline + опрашивается каждые 200 мс. +- Инварианты: + - **I1** — ready (или ожидаемый failed) не позже 60 с; + - **I2** — ровно 3 варианта ready-версии и ровно одно `succeeded` на job; + - **I3** — ready не откатывается, `ready_pipeline_version` не убывает; + - **I4** — подтверждённая работа не потеряна (есть успешный эффект); + - **I5** — нет дублей `(pipeline_version, variant)`; + - **I6** — SHA-256 вариантов совпадает с baseline для той же версии. + +### Результаты + +Таблица заполняется из `evidence/lab5/track2.json` и +`evidence/lab5/faults-level4.json`; фактические значения и пик памяти worker +приведены в `evidence/lab5/summary.md`. + +| Сценарий | Группа | Наблюдение (3 повтора для track2) | +| --- | --- | --- | +| `duplicate` | track2 | 5 повторных publish → 5 `duplicate_skipped` (`already_succeeded`); S3 etag/last_modified не изменились; attempt 1 | +| `reorder` | track2 | attempt 99 и дубль attempt 1 поглощены; ровно 2 skip, 1 `succeeded`; attempt 1 | +| `crash_before_side_effect` | track2 | worker убит до S3; попытка 2, `lease_expired`, процесс worker перезапущен | +| `crash_after_side_effect` | track2 | попытка 2, `lease_expired`, SHA-256 вариантов совпал с baseline (I6) | +| `crash_after_commit_before_ack` | track2 | попытка 1, один `succeeded`, redelivery → `duplicate_skipped` (`already_succeeded`) | +| `relay_crash_after_publish` | track2 | API убит между publish и mark; relay опубликовал снова, дубль поглощён; все 3 повтора | +| `stale_version` | track2 | `ready_pipeline_version=2`, v1 `superseded`, остались только варианты v2, v2 etag не изменились, served thumbnail SHA ≠ v1 | +| `crash_after_commit_before_publish` | level4 | API убит после COMMIT; outbox relay опубликовал после рестарта; ровно один `created`, ready | +| `retry_and_replay` | level4 | 4 `retry_scheduled`, затем `failed`/`retry_exhausted`; replay → attempt 6 → ready; `replayed` с operator | +| `hung_worker` | level4 | abort по `JOB_TIMEOUT_SECONDS`, `retry_scheduled` с `processing_timeout`, затем ready | +| `poison` | level4 | два невалидных сообщения ушли в DLQ, jobs не созданы | +| `big_image` | level4 | оба 40 Мп вложения ready, `OOMKilled=false`; пик 577–966 MiB при лимите 1280 MiB | + +Итог: `level4` — **7/7**, `track2` — **21/21** (7 сценариев × 3), все +инварианты I1–I6 истинны; см. `evidence/lab5/faults-level4.json`, +`evidence/lab5/track2.json`. + +### Ограничения эксперимента + +Это не exactly-once **доставка**: redelivery и повторная публикация наблюдаются. +S3-запись может повториться (объект перезаписывается тем же детерминированным +байтом), PostgreSQL — одна нода, а fault points находятся внутри процесса. +Сценарии `relay_crash_after_publish` и `crash_after_commit_before_publish` +зависят от гонки relay и поэтому фиксируют наблюдение, если дубль не успел +возникнуть. Внешние эффекты за пределами `(attachment, version, variant)` не +защищены. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Реализованы уровни 3–4 и трек 2; трек 1 (сравнение concurrency/prefetch) не + заявлен. +- Брокер и S3 — single-node; production-варианты требуют кворумов и ретраев + между зонами. +- `job_outbox` публикуется одним relay-процессом на реплику; при росте нагрузки + нужен отдельный relay или партиционирование. +- Ошибки классифицируются по сообщению sharp/`SERVER_FAULT`; для новых + форматов список стоит расширять по мере появления. +- Следующий шаг: метрики очереди lag и p95 ready, отдельный DLQ-replay UI, + сравнение concurrency/prefetch (трек 1). diff --git a/app/test/attachments.e2e-spec.ts b/app/test/attachments.e2e-spec.ts index d35bd73..e7bbcf8 100644 --- a/app/test/attachments.e2e-spec.ts +++ b/app/test/attachments.e2e-spec.ts @@ -20,6 +20,7 @@ import { postMessage, upload, uploadRaw, + waitReady, } from './support/e2e.js'; import sharp from 'sharp'; import { jpegWithMetadata, jpegWithoutScanData } from './support/jpeg.js'; @@ -55,8 +56,9 @@ async function ready(data: Buffer, type = 'image/png', chat = chatId) { const result = await upload(alice.token, chat, data, { contentType: type, }); - expect(result.response.status).toBe(201); - return result.body; + expect(result.response.status).toBe(202); + expect(result.body.status).toBe('queued'); + return waitReady(alice.token, result.body.id); } describe('attachments (proxy upload)', () => { @@ -76,27 +78,40 @@ describe('attachments (proxy upload)', () => { }); it('stores a PNG and serves it only to chat members', async () => { - const created = await upload(alice.token, chatId, png, { + const uploaded = await upload(alice.token, chatId, png, { filename: 'Снимок.png', }); - expect(created.response.status).toBe(201); - expect(created.body).toMatchObject({ + expect(uploaded.response.status).toBe(202); + expect(uploaded.body).toMatchObject({ chat_id: chatId, owner_id: alice.user.id, filename: 'Снимок.png', content_type: 'image/png', size_bytes: png.length, sha256: sha256(png), - status: 'ready', + status: 'queued', variants: [], error_code: null, }); - const id = created.body.id; + const id = uploaded.body.id; - const metadata = await http( - `/api/v1/attachments/${id}`, - { headers: bearer(bob.token) }, - ); + const ready = await waitReady(bob.token, id); + expect(ready).toMatchObject({ + filename: 'Снимок.png', + size_bytes: png.length, + sha256: sha256(png), + status: 'ready', + error_code: null, + }); + expect(ready.variants.map((variant) => variant.name)).toEqual([ + 'thumbnail', + 'crop', + 'optimized', + ]); + + const metadata = await http(`/api/v1/attachments/${id}`, { + headers: bearer(bob.token), + }); expect(metadata.response.status).toBe(200); expect(metadata.body.sha256).toBe(sha256(png)); @@ -241,7 +256,7 @@ describe('attachments (proxy upload)', () => { for (let index = 0; index < 4; index += 1) expect( (await upload(bob.token, chatId, png)).response.status, - ).toBe(201); + ).toBe(202); } finally { for (const socket of sockets) socket.destroy(); } @@ -346,7 +361,7 @@ describe('attachments (proxy upload)', () => { const created = await upload(alice.token, chatId, png, { filename: long, }); - expect(created.response.status).toBe(201); + expect(created.response.status).toBe(202); expect(created.body.filename).toBe(long); }); @@ -400,7 +415,7 @@ describe('attachments (proxy upload)', () => { it('rejects invalid attachment references', async () => { const mine = await ready(png); const foreign = await upload(eve.token, eveChatId, png); - expect(foreign.response.status).toBe(201); + expect(foreign.response.status).toBe(202); expectError( await postMessage(alice.token, chatId, { text: '' }), 400, @@ -439,8 +454,18 @@ describe('attachments (proxy upload)', () => { ); }); - it('serves unknown content variants as errors, not bytes', async () => { - const created = await ready(png); + it('serves the promised variants only after ready', async () => { + const uploaded = await upload(alice.token, chatId, png); + expect(uploaded.response.status).toBe(202); + expectError( + await http( + `/api/v1/attachments/${uploaded.body.id}/content`, + { headers: bearer(alice.token) }, + ), + 409, + 'conflict', + ); + const created = await waitReady(alice.token, uploaded.body.id); const variant = (name: string) => http( `/api/v1/attachments/${created.id}/content?variant=${name}`, @@ -449,7 +474,15 @@ describe('attachments (proxy upload)', () => { }, ); expect((await variant('original')).response.status).toBe(200); - expectError(await variant('thumbnail'), 404, 'not_found'); + for (const [name, type] of [ + ['thumbnail', 'image/jpeg'], + ['crop', 'image/jpeg'], + ['optimized', 'image/webp'], + ] as const) { + const response = await variant(name); + expect(response.response.status).toBe(200); + expect(response.response.headers.get('content-type')).toBe(type); + } expectError(await variant('bogus'), 400, 'invalid_request'); }); diff --git a/app/test/dependency-failures.e2e-spec.ts b/app/test/dependency-failures.e2e-spec.ts index 7f489cc..9b274f5 100644 --- a/app/test/dependency-failures.e2e-spec.ts +++ b/app/test/dependency-failures.e2e-spec.ts @@ -13,6 +13,7 @@ import type { Page } from '../src/common/http/page.js'; import type { Message } from '../src/messages/message.js'; import type { User } from '../src/users/user.js'; import { noisePng } from './support/png.js'; +import { waitReady } from './support/e2e.js'; // Automated counterpart of the manual Redis/PostgreSQL outage checks. Four // topologies, because the load balancer probes /health/ready (the same route as @@ -924,7 +925,8 @@ describe.skipIf(!enabled)('dependency outages', () => { }, ); const before = await upload(samplePng, `s3before-${runId}.png`); - expect(before.response.status).toBe(201); + expect(before.response.status).toBe(202); + await waitReady(token, before.body.id); await compose('stop', s3Service); try { @@ -969,7 +971,8 @@ describe.skipIf(!enabled)('dependency outages', () => { Buffer.from(await bytes.arrayBuffer()).equals(samplePng), ).toBe(true); const after = await upload(samplePng, `s3after-${runId}.png`); - expect(after.response.status).toBe(201); + expect(after.response.status).toBe(202); + await waitReady(token, after.body.id); // A `deleting` row is re-claimed one minute after its last change // (an in-flight compensation is never raced), then swept. @@ -1001,7 +1004,8 @@ describe.skipIf(!enabled)('dependency outages', () => { body: samplePng as unknown as BodyInit, }, ); - expect(created.response.status).toBe(201); + expect(created.response.status).toBe(202); + await waitReady(token, created.body.id); // A paused container keeps its TCP port open but never answers: the // failure mode a stopped container (connection refused) does not show. const container = (await compose('ps', '-q', s3Service)).split( @@ -1084,9 +1088,9 @@ describe.skipIf(!enabled)('dependency outages', () => { await waitForReadiness(200); const key = await psql( - `select object_key from attachments where ${filter}`, + `select staging_key from attachments where ${filter}`, ); - expect(key).toMatch(/^attachments\/[0-9a-f-]{36}\/original$/u); + expect(key).toMatch(/^staging\/[0-9a-f-]{36}\/original$/u); // Without PostgreSQL the API cannot tell a lost commit from a failed // one, so it deletes nothing; the pending row keeps the object tracked. expect(await objectExists(key)).toBe(true); diff --git a/app/test/direct-upload.e2e-spec.ts b/app/test/direct-upload.e2e-spec.ts index 7713bdb..04149a4 100644 --- a/app/test/direct-upload.e2e-spec.ts +++ b/app/test/direct-upload.e2e-spec.ts @@ -17,6 +17,7 @@ import { http, json, postMessage, + waitReady, } from './support/e2e.js'; import { noisePng, pixelBombPng } from './support/png.js'; @@ -102,7 +103,7 @@ describe('direct upload (presigned PUT)', () => { ); expect(Date.parse(ticket.expires_at)).toBeGreaterThan(Date.now()); expect(new URL(ticket.url).pathname).toBe( - `/messenger-attachments/attachments/${ticket.attachment_id}/original`, + `/messenger-attachments/staging/${ticket.attachment_id}/original`, ); const hidden = await http( @@ -143,11 +144,14 @@ describe('direct upload (presigned PUT)', () => { content_type: 'image/png', size_bytes: png.length, sha256: sha256(png), - status: 'ready', + status: 'queued', }); + const ready = await waitReady(alice.token, ticket.attachment_id); + expect(ready.status).toBe('ready'); + expect(ready.variants).toHaveLength(3); const again = await complete(alice.token, ticket.attachment_id); expect(again.response.status).toBe(200); - expect(again.body).toEqual(done.body); + expect(again.body.status).toBe('ready'); const content = await download(bob.token, ticket.attachment_id); expect(content.response.status).toBe(200); diff --git a/app/test/support/e2e.ts b/app/test/support/e2e.ts index 5210baa..7f0338a 100644 --- a/app/test/support/e2e.ts +++ b/app/test/support/e2e.ts @@ -134,6 +134,35 @@ export function upload( ); } +/** + * Lab5 uploads answer 202 before the worker runs. Poll metadata until ready; + * a failed attachment is a test failure, never a silent timeout. + */ +export async function waitReady( + token: string, + attachmentId: string, + timeoutMs = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const result = await http( + `/api/v1/attachments/${attachmentId}`, + { headers: bearer(token) }, + ); + expect(result.response.status).toBe(200); + if (result.body.status === 'failed') + throw new Error( + `attachment ${attachmentId} failed: ${result.body.error_code ?? 'unknown'}`, + ); + if (result.body.status === 'ready') return result.body; + if (Date.now() >= deadline) + throw new Error( + `attachment ${attachmentId} did not become ready within ${timeoutMs}ms`, + ); + await new Promise((resolve) => setTimeout(resolve, 200)); + } +} + export async function download( token: string, attachmentId: string, diff --git a/evidence/lab5/big-image-memory.txt b/evidence/lab5/big-image-memory.txt new file mode 100644 index 0000000..85c08ac --- /dev/null +++ b/evidence/lab5/big-image-memory.txt @@ -0,0 +1,4 @@ +big_image: peak worker memory: 577.0 MiB / 1280 MiB +invariants: {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} +duration_s: 22.697 +peak accounting: two worst_decode_png (8000x5000) uploaded concurrently; worker limit 1280 MiB; OOMKilled=false diff --git a/evidence/lab5/build-lint.txt b/evidence/lab5/build-lint.txt new file mode 100644 index 0000000..b5d76a1 --- /dev/null +++ b/evidence/lab5/build-lint.txt @@ -0,0 +1,8 @@ + +> app@0.0.1 build +> nest build + + +> app@0.0.1 lint +> oxlint src/ test/ + diff --git a/evidence/lab5/compose-ps.txt b/evidence/lab5/compose-ps.txt new file mode 100644 index 0000000..02c4e5d --- /dev/null +++ b/evidence/lab5/compose-ps.txt @@ -0,0 +1,9 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab5-api-1 mtusi-messenger:lab5 "docker-entrypoint.s…" api 6 minutes ago Up 37 seconds (healthy) 8080/tcp +mtusi-lab5-api-2 mtusi-messenger:lab5 "docker-entrypoint.s…" api 6 minutes ago Up 37 seconds (healthy) 8080/tcp +mtusi-lab5-broker-1 rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e "docker-entrypoint.s…" broker 56 minutes ago Up 39 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-lab5-haproxy-1 haproxy:3.2-alpine "docker-entrypoint.s…" haproxy 56 minutes ago Up About a minute 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 56 minutes ago Up About a minute (healthy) 127.0.0.1:5432->5432/tcp +mtusi-lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 56 minutes ago Up About a minute (healthy) 127.0.0.1:6379->6379/tcp +mtusi-lab5-s3-1 minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e "/usr/bin/docker-ent…" s3 56 minutes ago Up About a minute (healthy) 127.0.0.1:9000->9000/tcp +mtusi-lab5-worker-1 mtusi-messenger:lab5 "docker-entrypoint.s…" worker 6 minutes ago Up 5 seconds (healthy) 8080/tcp diff --git a/evidence/lab5/failover.txt b/evidence/lab5/failover.txt new file mode 100644 index 0000000..9790ef2 --- /dev/null +++ b/evidence/lab5/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 3.474} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5/faults-level4.json b/evidence/lab5/faults-level4.json new file mode 100644 index 0000000..5b74211 --- /dev/null +++ b/evidence/lab5/faults-level4.json @@ -0,0 +1,138 @@ +[ + { + "name": "crash_after_commit_before_publish", + "repeat": 1, + "passed": true, + "duration_s": 16.402, + "attachment_id": "5a030f81-3967-43aa-94ba-3ef24f043efe", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "outbox relay published the committed job after API restart" + }, + { + "name": "crash_after_side_effect", + "repeat": 1, + "passed": true, + "duration_s": 42.306, + "attachment_id": "fd01701f-635e-43d9-b41a-86cfbf654af1", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_side_effect" + }, + { + "name": "crash_after_commit_before_ack", + "repeat": 1, + "passed": true, + "duration_s": 33.299, + "attachment_id": "8e1752c9-e073-4d0a-b858-fff8fbc156ff", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_commit_before_ack" + }, + { + "name": "retry_and_replay", + "repeat": 1, + "passed": true, + "duration_s": 44.868, + "attachment_id": "721aa545-6d6d-4c5e-a353-dc8e39751442", + "job_attempts": [ + 6 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "transient injected fault exhausted retries; replay recovered" + }, + { + "name": "hung_worker", + "repeat": 1, + "passed": true, + "duration_s": 44.634, + "attachment_id": "fb451ee8-cf2c-423a-9e22-a95ec00d58fd", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "hard timeout aborted the hung render and retried" + }, + { + "name": "poison", + "repeat": 1, + "passed": true, + "duration_s": 22.962, + "attachment_id": null, + "job_attempts": [], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "malformed messages moved to the DLQ without touching jobs" + }, + { + "name": "big_image", + "repeat": 1, + "passed": true, + "duration_s": 22.697, + "attachment_id": "9eae2f87-64e0-4e27-bdf5-cc04b77c4a8d", + "job_attempts": [], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "peak worker memory: 577.0 MiB / 1280 MiB" + } +] diff --git a/evidence/lab5/faults-level4.txt b/evidence/lab5/faults-level4.txt new file mode 100644 index 0000000..92a4dc5 --- /dev/null +++ b/evidence/lab5/faults-level4.txt @@ -0,0 +1,9 @@ +python3 scripts/lab5/fault_suite.py --base-url "http://localhost:8080" --scenario level4 --output evidence/lab5/faults-level4.json +PASS crash_after_commit_before_publish #1 16.402s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} outbox relay published the committed job after API restart +PASS crash_after_side_effect #1 42.306s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} crashed at worker.after_side_effect +PASS crash_after_commit_before_ack #1 33.299s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} crashed at worker.after_commit_before_ack +PASS retry_and_replay #1 44.868s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} transient injected fault exhausted retries; replay recovered +PASS hung_worker #1 44.634s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} hard timeout aborted the hung render and retried +PASS poison #1 22.962s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} malformed messages moved to the DLQ without touching jobs +PASS big_image #1 22.697s {"I1": true, "I2": true, "I3": true, "I4": true, "I5": true, "I6": true} peak worker memory: 577.0 MiB / 1280 MiB +7/7 scenarios passed diff --git a/evidence/lab5/make-check.txt b/evidence/lab5/make-check.txt new file mode 100644 index 0000000..9a55330 --- /dev/null +++ b/evidence/lab5/make-check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/lab5/make-test-dependencies.txt b/evidence/lab5/make-test-dependencies.txt new file mode 100644 index 0000000..0a83bef --- /dev/null +++ b/evidence/lab5/make-test-dependencies.txt @@ -0,0 +1,15 @@ +cd app && E2E_COMPOSE_DIR="/home/tomato/Projects/VP/mtusi2026lab5" E2E_COMPOSE_ENV_FILE=".env" npm run test:dependencies + +> app@0.0.1 test:dependencies +> E2E_DEPENDENCY_FAULTS=1 vitest run --config ./vitest.config.e2e.ts dependency-failures + +The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead. + + RUN v4.1.11 /home/tomato/Projects/VP/mtusi2026lab5/app + + + Test Files 1 passed (1) + Tests 7 passed (7) + Start at 01:10:32 + Duration 165.18s (transform 316ms, setup 0ms, import 378ms, tests 164.50s, environment 0ms) + diff --git a/evidence/lab5/make-test-e2e.txt b/evidence/lab5/make-test-e2e.txt new file mode 100644 index 0000000..a60d7b6 --- /dev/null +++ b/evidence/lab5/make-test-e2e.txt @@ -0,0 +1,15 @@ +scripts/app-e2e.sh + +> app@0.0.1 test:e2e +> vitest run --config ./vitest.config.e2e.ts + +The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead. + + RUN v4.1.11 /home/tomato/Projects/VP/mtusi2026lab5/app + + + Test Files 3 passed | 1 skipped (4) + Tests 44 passed | 9 skipped (53) + Start at 01:10:05 + Duration 22.44s (transform 858ms, setup 0ms, import 2.30s, tests 36.01s, environment 4ms) + diff --git a/evidence/lab5/make-test.txt b/evidence/lab5/make-test.txt new file mode 100644 index 0000000..df05fe2 --- /dev/null +++ b/evidence/lab5/make-test.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5/persistence.txt b/evidence/lab5/persistence.txt new file mode 100644 index 0000000..793b001 --- /dev/null +++ b/evidence/lab5/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5/queues.txt b/evidence/lab5/queues.txt new file mode 100644 index 0000000..b7c992e --- /dev/null +++ b/evidence/lab5/queues.txt @@ -0,0 +1,3 @@ +{"process":{"messages":0,"consumers":1},"dlq":{"messages":10,"consumers":0}} + +{"attachment":{"id":"6920d20c-b819-4d20-b922-ded8cd136467","status":"ready","error_code":null,"ready_pipeline_version":1,"staging_present":false},"jobs":[{"job_id":"da217f72-cc86-4621-a595-633821709c6a","pipeline_version":1,"status":"succeeded","attempt":1,"max_attempts":5,"last_error_code":null}],"variants":[{"pipeline_version":1,"variant":"thumbnail","sha256":"a18bfc29cd9311807ce56afc7078d45db13b19ac2ac14a31d2b91f2640eddf29","size_bytes":4101,"width":256,"height":160,"object_key":"attachments/6920d20c-b819-4d20-b922-ded8cd136467/v1/thumbnail.jpg","s3":{"exists":true,"etag":"\"1c094bdd7c6cf854d0c4429768fb8c49\"","last_modified":"2026-11-04T22:15:04.000Z"}},{"pipeline_version":1,"variant":"crop","sha256":"06c85b2c7a64a45c5e88d65ef492c3e54bc3a6dd1c4a2fad1dccd71b9aba8113","size_bytes":1657,"width":128,"height":128,"object_key":"attachments/6920d20c-b819-4d20-b922-ded8cd136467/v1/crop.jpg","s3":{"exists":true,"etag":"\"f36ea628d9691b1d7fbad35499d3863f\"","last_modified":"2026-11-04T22:15:04.000Z"}},{"pipeline_version":1,"variant":"optimized","sha256":"0bb5b26524335872b2dfe977e2e364101870bfa4e223f4383ab00c34f6771735","size_bytes":2148,"width":320,"height":200,"object_key":"attachments/6920d20c-b819-4d20-b922-ded8cd136467/v1/optimized.webp","s3":{"exists":true,"etag":"\"03817f0652c967e9a963d9b29a1152c1\"","last_modified":"2026-11-04T22:15:04.000Z"}}],"original":{"exists":true,"etag":"\"c2ab21400d8c13ea847e4d5d3c9832ad\"","last_modified":"2026-11-04T22:15:04.000Z"},"events":[{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"created","attempt":1,"detail":{},"created_at":"2026-11-04T22:14:18.736Z"},{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"claimed","attempt":1,"detail":{"lease_owner":"11fa090ed849"},"created_at":"2026-11-04T22:15:04.465Z"},{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"succeeded","attempt":1,"detail":{},"created_at":"2026-11-04T22:15:04.599Z"},{"job_id":"da217f72-cc86-4621-a595-633821709c6a","event":"staging_deleted","attempt":1,"detail":{"staging_key":"staging/6920d20c-b819-4d20-b922-ded8cd136467/original"},"created_at":"2026-11-04T22:15:04.615Z"}],"outbox":{"pending":0,"published":1}} diff --git a/evidence/lab5/summary.md b/evidence/lab5/summary.md new file mode 100644 index 0000000..d5e9956 --- /dev/null +++ b/evidence/lab5/summary.md @@ -0,0 +1,41 @@ +# lab5 — сводка доказательств + +Дата прогона: 2026-11-05. Хост: 8 vCPU i7-11370H, 15 GiB RAM; Docker 29.9.2, +Node 25.7.0, Python 3.14.8. Все команды выполнены из корня репозитория; секреты +и токены не печатались. + +| Критерий | Файл | Наблюдение | +| --- | --- | --- | +| Offline-структура и OpenAPI | `make-check.txt` | `PASS: offline skeleton structure` | +| Публичный smoke (уровень 3) | `make-test.txt` | `PASS: lab5 public API scenario`; 202 → ready → thumbnail 256×160 JPEG | +| Юнит-тесты | `unit-tests.txt` | 18 файлов, 69 тестов, все зелёные; включены fit-inside, backoff, job-message, classifyError, pipelines (EXIF, alpha, determinism) | +| E2E приложения | `make-test-e2e.txt` | 44 passed, 9 skipped (opt-in dependency сценарии) | +| Отказы зависимостей | `make-test-dependencies.txt` | 7/7: Redis, PostgreSQL, per-replica DB path, насыщение пула, S3 stop/pause, staging при PG down | +| Сохранность после рестарта stores | `persistence.txt` | `PASS: persistence preserved confirmed data and existing session` | +| Failover API-реплики | `failover.txt` | 124 успешных записи, 0 ошибок, первое восстановление 3.474 s, 2 инстанса до отказа | +| Демонстрация «worker остановлен» | `worker-down-demo.txt` | 202 queued, content 409, сообщение привязано, очередь пережила restart broker+API, после start worker — ready и thumbnail 256×160 | +| Уровень 4, fault injection | `faults-level4.json`, `faults-level4.txt` | 7/7 сценариев, все I1–I6; crash COMMIT→publish, retry/replay, timeout, DLQ, big image | +| Трек 2, 3 повтора | `track2.json`, `track2.txt` | 21/21 (7 сценариев × 3), все I1–I6; duplicate, reorder, crash до/после side effect, crash после COMMIT до ack, relay crash, stale version | +| Память worker на 40 Мп | `big-image-memory.txt` | 2 × 8000×5000 одновременно; пик 577–966 MiB (замеры разных прогонов) при лимите 1280 MiB; `OOMKilled=false` | +| Состав стенда | `compose-ps.txt` | api ×2 healthy, worker healthy, broker healthy, postgres/redis/s3 healthy | +| Очереди и job после demo | `queues.txt` | process 0/1 consumer, DLQ 10 (накопленные poison-сообщения); job succeeded attempt 1, `staging_present=false` | + +## Параметры fault-стенда + +```sh +FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 JOB_LEASE_SECONDS=8 \ +JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 make stand-up +``` + +`WORKER_CONCURRENCY=2`, `JOB_MAX_ATTEMPTS=5`, `WORKER_MEMORY=1280m`. +После экспериментов стенд возвращён в штатный режим +(`FAULT_INJECTION_ENABLED=0`, timeout 60 s, lease 30 s) и `make test` повторён. + +## Итог трека 2 + +`track2.json`: 21 запуск, 21 passed. Средние длительности: duplicate 16.7 s, +reorder 36.7 s, crash_before 42.1 s, crash_after 43.9 s, crash_after_ack +33.2 s, relay_crash 42.1 s, stale_version 48.8 s. Ни один инвариант I1–I6 не +нарушен. Повторные публикации и redelivery наблюдались и поглощались +(`duplicate_skipped`), S3 etag вариантов не менялся; v1 после v2 получил +`superseded`, а v2-объекты остались нетронутыми. diff --git a/evidence/lab5/track2.json b/evidence/lab5/track2.json new file mode 100644 index 0000000..3abb875 --- /dev/null +++ b/evidence/lab5/track2.json @@ -0,0 +1,425 @@ +[ + { + "name": "duplicate", + "repeat": 1, + "passed": true, + "duration_s": 16.515, + "attachment_id": "74730f7b-2803-4da3-a430-7097dbe6a799", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "5 duplicate deliveries were absorbed by the DB fence" + }, + { + "name": "duplicate", + "repeat": 2, + "passed": true, + "duration_s": 16.938, + "attachment_id": "26d8ab29-e056-4ea0-ab45-2861539515bd", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "5 duplicate deliveries were absorbed by the DB fence" + }, + { + "name": "duplicate", + "repeat": 3, + "passed": true, + "duration_s": 16.652, + "attachment_id": "f6d5afa2-5e36-4444-8e64-b13c4da41d20", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "5 duplicate deliveries were absorbed by the DB fence" + }, + { + "name": "reorder", + "repeat": 1, + "passed": true, + "duration_s": 37.233, + "attachment_id": "420cc224-9dcd-4e92-90c3-126d83aaf640", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "attempt 99 and a duplicate attempt 1 were acked as skips" + }, + { + "name": "reorder", + "repeat": 2, + "passed": true, + "duration_s": 36.546, + "attachment_id": "4dbea2c4-534b-475b-952c-4d1af3e1b9fd", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "attempt 99 and a duplicate attempt 1 were acked as skips" + }, + { + "name": "reorder", + "repeat": 3, + "passed": true, + "duration_s": 36.47, + "attachment_id": "3d56f2f1-a09a-48aa-b605-ab5a69c882e8", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "attempt 99 and a duplicate attempt 1 were acked as skips" + }, + { + "name": "crash_before_side_effect", + "repeat": 1, + "passed": true, + "duration_s": 40.677, + "attachment_id": "9d85c5fa-4da8-47ba-a986-1c39b49a27a1", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.before_side_effect" + }, + { + "name": "crash_before_side_effect", + "repeat": 2, + "passed": true, + "duration_s": 42.92, + "attachment_id": "3d4b7a4a-d1e6-4ced-9f59-70bac8b8dfec", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.before_side_effect" + }, + { + "name": "crash_before_side_effect", + "repeat": 3, + "passed": true, + "duration_s": 42.797, + "attachment_id": "0cda94bc-19b7-454a-8898-2bfd11facb6c", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.before_side_effect" + }, + { + "name": "crash_after_side_effect", + "repeat": 1, + "passed": true, + "duration_s": 44.231, + "attachment_id": "6ade96a3-f2dd-413d-9f0d-ff49fcfcc3bd", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_side_effect" + }, + { + "name": "crash_after_side_effect", + "repeat": 2, + "passed": true, + "duration_s": 43.819, + "attachment_id": "08ed488b-28a9-4971-845d-05a8744546fd", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_side_effect" + }, + { + "name": "crash_after_side_effect", + "repeat": 3, + "passed": true, + "duration_s": 43.639, + "attachment_id": "d870e2b8-d11b-47b4-b200-545f731fccaf", + "job_attempts": [ + 2 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_side_effect" + }, + { + "name": "crash_after_commit_before_ack", + "repeat": 1, + "passed": true, + "duration_s": 32.458, + "attachment_id": "667b757b-3e78-4e61-8326-50d2fe206932", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_commit_before_ack" + }, + { + "name": "crash_after_commit_before_ack", + "repeat": 2, + "passed": true, + "duration_s": 32.862, + "attachment_id": "f51e94d4-da83-45cf-b252-fe06409a70a8", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_commit_before_ack" + }, + { + "name": "crash_after_commit_before_ack", + "repeat": 3, + "passed": true, + "duration_s": 34.177, + "attachment_id": "fda75d45-b9dc-4a66-9901-081283d77fa3", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "crashed at worker.after_commit_before_ack" + }, + { + "name": "relay_crash_after_publish", + "repeat": 1, + "passed": true, + "duration_s": 42.734, + "attachment_id": "004d345c-47cb-4905-96f1-48ebe6a36b9a", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "relay republished the same message; duplicate absorbed" + }, + { + "name": "relay_crash_after_publish", + "repeat": 2, + "passed": true, + "duration_s": 40.802, + "attachment_id": "5a0a97ec-d0d4-4015-ac96-99fe1346cee4", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "relay republished the same message; duplicate absorbed" + }, + { + "name": "relay_crash_after_publish", + "repeat": 3, + "passed": true, + "duration_s": 42.804, + "attachment_id": "8ee159d6-786f-434d-be02-4ec0cedfb898", + "job_attempts": [ + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "relay republished the same message; duplicate absorbed" + }, + { + "name": "stale_version", + "repeat": 1, + "passed": true, + "duration_s": 49.303, + "attachment_id": "c4be0676-10b0-4249-881e-2d28eaed812f", + "job_attempts": [ + 1, + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "v1 finished after v2 and was recorded as superseded" + }, + { + "name": "stale_version", + "repeat": 2, + "passed": true, + "duration_s": 48.162, + "attachment_id": "76a942c6-fc00-4fc7-a94c-72a5c1298022", + "job_attempts": [ + 1, + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "v1 finished after v2 and was recorded as superseded" + }, + { + "name": "stale_version", + "repeat": 3, + "passed": true, + "duration_s": 48.812, + "attachment_id": "3c2675b5-f301-4b80-8448-c4fa7067d082", + "job_attempts": [ + 1, + 1 + ], + "events_summary": {}, + "invariants": { + "I1": true, + "I2": true, + "I3": true, + "I4": true, + "I5": true, + "I6": true + }, + "notes": "v1 finished after v2 and was recorded as superseded" + } +] diff --git a/evidence/lab5/track2.txt b/evidence/lab5/track2.txt new file mode 100644 index 0000000..e54fc8f --- /dev/null +++ b/evidence/lab5/track2.txt @@ -0,0 +1,22 @@ +track2: 21/21 scenarios passed +PASS duplicate #1 16.515s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence +PASS duplicate #2 16.938s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence +PASS duplicate #3 16.652s I1=true I2=true I3=true I4=true I5=true I6=true 5 duplicate deliveries were absorbed by the DB fence +PASS reorder #1 37.233s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips +PASS reorder #2 36.546s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips +PASS reorder #3 36.470s I1=true I2=true I3=true I4=true I5=true I6=true attempt 99 and a duplicate attempt 1 were acked as skips +PASS crash_before_side_effect #1 40.677s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect +PASS crash_before_side_effect #2 42.920s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect +PASS crash_before_side_effect #3 42.797s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.before_side_effect +PASS crash_after_side_effect #1 44.231s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect +PASS crash_after_side_effect #2 43.819s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect +PASS crash_after_side_effect #3 43.639s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_side_effect +PASS crash_after_commit_before_ack #1 32.458s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack +PASS crash_after_commit_before_ack #2 32.862s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack +PASS crash_after_commit_before_ack #3 34.177s I1=true I2=true I3=true I4=true I5=true I6=true crashed at worker.after_commit_before_ack +PASS relay_crash_after_publish #1 42.734s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed +PASS relay_crash_after_publish #2 40.802s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed +PASS relay_crash_after_publish #3 42.804s I1=true I2=true I3=true I4=true I5=true I6=true relay republished the same message; duplicate absorbed +PASS stale_version #1 49.303s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded +PASS stale_version #2 48.162s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded +PASS stale_version #3 48.812s I1=true I2=true I3=true I4=true I5=true I6=true v1 finished after v2 and was recorded as superseded diff --git a/evidence/lab5/unit-tests.txt b/evidence/lab5/unit-tests.txt new file mode 100644 index 0000000..a983021 --- /dev/null +++ b/evidence/lab5/unit-tests.txt @@ -0,0 +1,17 @@ + +> app@0.0.1 test:unit +> vitest run src/ + +The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead. + + RUN v4.1.11 /home/tomato/Projects/VP/mtusi2026lab5/app + +[Nest] 1482602 - 11/05/2026, 1:17:43 AM WARN [MessagesCacheService] Cache read failed, serving PostgreSQL: redis down +[Nest] 1482602 - 11/05/2026, 1:17:43 AM WARN [MessagesCacheService] Cache fill failed: redis down +[Nest] 1482996 - 11/05/2026, 1:17:50 AM WARN [AttachmentsService] Compensation of attachment e7f89989-b925-4ffb-ae59-9e26e897b963 deferred: Connection terminated unexpectedly + + Test Files 18 passed (18) + Tests 69 passed (69) + Start at 01:17:39 + Duration 10.17s (transform 3.18s, setup 0ms, import 32.82s, tests 7.20s, environment 3ms) + diff --git a/evidence/lab5/worker-down-demo.txt b/evidence/lab5/worker-down-demo.txt new file mode 100644 index 0000000..1318443 --- /dev/null +++ b/evidence/lab5/worker-down-demo.txt @@ -0,0 +1,36 @@ +python3 scripts/lab5/worker_down_demo.py --base-url "http://localhost:8080" +[1] compose ps +SERVICE STATUS +api Up 50 seconds (healthy) +api Up 4 seconds (health: starting) +broker Up 53 seconds (healthy) +haproxy Up 54 seconds +postgres Up 54 seconds (healthy) +redis Up 54 seconds (healthy) +s3 Up 54 seconds (healthy) +worker Up 54 seconds (healthy) +[2] stop worker +[3] upload while worker is down (expect 202 queued) +attachment=6920d20c-b819-4d20-b922-ded8cd136467 status=queued +[4] content and thumbnail must answer 409 before ready +[5] link the queued attachment to messages +message=7b4ec662-3361-4210-b62c-da913537d21e +[6] wait for outbox publish and a message in the queue +queue={'messages': 1, 'type': 'quorum'} +[7] restart broker and API with volumes +[8] the quorum message survived the broker restart +queue={'messages': 1, 'type': 'quorum'} +[9] start worker +[10] wait ready and verify the thumbnail +ready attachment=6920d20c-b819-4d20-b922-ded8cd136467 thumbnail=256x160 sha256=a18bfc29cd931180... +[11] compose ps +SERVICE STATUS +api Up 33 seconds (healthy) +api Up 33 seconds (healthy) +broker Up 35 seconds (healthy) +haproxy Up About a minute +postgres Up About a minute (healthy) +redis Up About a minute (healthy) +s3 Up About a minute (healthy) +worker Up 2 seconds (health: starting) +PASS: worker-down durability demo diff --git a/infra/README.md b/infra/README.md index 5a39e86..cd000a0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -2,6 +2,29 @@ Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. + +## Роли lab5 + +| Роль | Service | Образ | Готовность | Volume | Credentials | +| --- | --- | --- | --- | --- | --- | +| API | `api` | сборка `mtusi-messenger:lab5` | `/health/ready` (PostgreSQL, Redis, S3) | — | `secrets/s3_api_*`, `secrets/broker_url` | +| Worker | `worker` | та же сборка, `dist/worker.main.js` | `127.0.0.1:8081/health` (consumer активен) | — | `secrets/s3_worker_*`, `secrets/broker_url` | +| Broker | `broker` | `rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e` | `rabbitmq-diagnostics check_port_connectivity` | `broker-data` | `secrets/rabbitmq.conf` | +| PostgreSQL | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `postgres-data` | пароль в `.env` / `DATABASE_URL` | +| Redis | `redis` | `redis:7.4.2-alpine` | `redis-cli ping` | `redis-data` | без пароля, только compose-сеть | +| Object storage | `s3` | `minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e` | `mc ready local` | `s3-data` | root — `secrets/minio_root_*` | +| S3 bootstrap | `s3-init` | `minio/mc:RELEASE.2025-08-13T08-35-41Z@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727` | завершается 0 | — | root + identities API/worker | +| Load balancer | `haproxy` | `haproxy:3.2-alpine` | зависит от `api` healthy | — | — | + +Политики MinIO разделены: API пишет/удаляет только `staging/*`, читает и удаляет +`attachments/*`; worker читает/удаляет `staging/*` и читает/пишет `attachments/*`. +Обе роли получают `s3:ListBucket`. Пароль брокера генерируется один раз в +`scripts/init-env.sh` и попадает и в `secrets/broker_url` (amqplib), и в +`secrets/rabbitmq.conf` (сам брокер), поэтому URL и конфигурация не расходятся. + +Bootstrap идемпотентен: `scripts/init-env.sh` создаёт `.env` и секреты, `migrate` +применяет SQL-миграции, `s3-init` создаёт bucket/политики/пользователей, а +producer и consumer объявляют обменники и quorum-очереди при каждом подключении. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. diff --git a/scripts/lab5/fault_suite.py b/scripts/lab5/fault_suite.py new file mode 100755 index 0000000..ae02145 --- /dev/null +++ b/scripts/lab5/fault_suite.py @@ -0,0 +1,749 @@ +#!/usr/bin/env python3 +"""Automated fault injection for lab5 levels 4 and 5 (track 2).""" +import argparse +import json +import re +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4')) + +from stand import ( # noqa: E402 + ROOT, + Failure, + check, + clear_fault, + clear_faults, + cli, + compose, + docker, + new_client, + rabbit_queues, + sample_bytes, + set_fault, + sha256, + upload, + variant_sha, + wait_for, + wait_ready, + worker_env, + worker_start_count, +) +from pngkit import worst_decode_png # noqa: E402 + +HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 ' + 'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 ' + 'make stand-up') + +LEVEL4 = [ + 'crash_after_commit_before_publish', + 'crash_after_side_effect', + 'crash_after_commit_before_ack', + 'retry_and_replay', + 'hung_worker', + 'poison', + 'big_image', +] +TRACK2 = [ + 'duplicate', + 'reorder', + 'crash_before_side_effect', + 'crash_after_side_effect', + 'crash_after_commit_before_ack', + 'relay_crash_after_publish', + 'stale_version', +] + + +class Tracker(threading.Thread): + """Polls metadata while a scenario runs to prove readiness never regresses.""" + + def __init__(self, base_url, actor, ca_file=None): + super().__init__(daemon=True) + self.client = new_client(base_url, ca_file) + self.actor = actor + self.ids = [] + self.records = {} + self.stop_event = threading.Event() + + def track(self, attachment_id): + if attachment_id not in self.records: + self.ids.append(attachment_id) + self.records[attachment_id] = [] + + def run(self): + while not self.stop_event.is_set(): + for attachment_id in list(self.ids): + try: + meta = self.client.call( + 'GET', '/api/v1/attachments/' + attachment_id, + actor=self.actor, schema='Attachment', + )['json'] + self.records[attachment_id].append( + (time.monotonic(), meta['status'], + meta.get('ready_pipeline_version')) + ) + except Exception: + pass + self.stop_event.wait(0.2) + + def stop(self): + self.stop_event.set() + self.join(timeout=5) + + +def build_message(status, job, attempt=None): + created = next( + event for event in status['events'] + if event['job_id'] == job['job_id'] and event['event'] == 'created' + ) + return { + 'schema_version': 1, + 'job_id': job['job_id'], + 'type': 'attachment.process.v1', + 'attachment_id': status['attachment']['id'], + 'pipeline_version': job['pipeline_version'], + 'attempt': attempt if attempt is not None else job['attempt'], + 'created_at': created['created_at'], + } + + +def publish_raw(messages): + text = ''.join(json.dumps(message, separators=(',', ':')) + '\n' + for message in messages) + code, payload = cli('publish-raw', stdin=text) + check(code == 0, 'publish-raw failed: ' + str(payload)) + return payload + + +def status_of(attachment_id): + code, payload = cli('status', attachment_id) + check(code == 0, 'status failed: ' + str(payload)) + return payload + + +def flush_outbox(attachment_id): + wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0, + 30, 'outbox publish', 1.0) + + +def new_attachment_id(chat_id, before): + rows = cli('find-by-chat', chat_id)[1] + for row in reversed(rows): + if row['id'] not in before: + return row['id'] + return None + + +def evaluate(tracker, attachment_id, baseline, expect_failed=False, + compare_baseline=True): + if attachment_id is None: + return {'I1': True, 'I2': True, 'I3': True, + 'I4': True, 'I5': True, 'I6': True} + status = status_of(attachment_id) + attachment = status['attachment'] + jobs = status['jobs'] + variants = status['variants'] + ready_version = attachment['ready_pipeline_version'] + + records = tracker.records.get(attachment_id, []) + reached = any(record[1] == ('failed' if expect_failed else 'ready') + for record in records) + if not records: + reached = attachment['status'] == ('failed' if expect_failed else 'ready') + i1 = reached or attachment['status'] in ('ready', 'failed') + + ready_variants = [v for v in variants + if v['pipeline_version'] == ready_version] + succeeded_ids = {job['job_id'] for job in jobs + if job['status'] == 'succeeded'} + succeeded_events = {} + for event in status['events']: + if event['event'] == 'succeeded': + succeeded_events[event['job_id']] = ( + succeeded_events.get(event['job_id'], 0) + 1 + ) + i2 = (len(ready_variants) == 3 + and all(succeeded_events.get(job_id) == 1 + for job_id in succeeded_ids)) + + i3 = True + seen_ready = False + last_version = 0 + for _at, state, version in records: + if seen_ready and state != 'ready': + i3 = False + if state == 'ready': + seen_ready = True + if version is not None: + if version < last_version: + i3 = False + last_version = version + + i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready' + keys = {(variant['pipeline_version'], variant['variant']) + for variant in variants} + i5 = len(keys) == len(variants) + + if compare_baseline and ready_version == 1 and ready_variants: + observed = {variant['variant']: variant['sha256'] + for variant in ready_variants} + i6 = all(observed.get(name) == digest + for name, digest in baseline.items()) + else: + i6 = True + + return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4, + 'I5': i5, 'I6': i6} + + +# --- scenarios ----------------------------------------------------------- + + +def scenario_duplicate(ctx): + attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], + sample_bytes(), 'duplicate.png') + ctx['attachment_id'] = attachment['id'] + ctx['tracker'].track(attachment['id']) + wait_ready(ctx['client'], ctx['alice'], attachment['id']) + status = status_of(attachment['id']) + before = {variant['variant']: variant['s3'] + for variant in status['variants']} + message = build_message(status, status['jobs'][0]) + publish_raw([message] * 5) + wait_for( + lambda: len([event for event in status_of(attachment['id'])['events'] + if event['event'] == 'duplicate_skipped' + and event['detail'].get('reason') == 'already_succeeded']) == 5, + 20, 'five duplicate_skipped events', 1.0, + ) + after_status = status_of(attachment['id']) + after = {variant['variant']: variant['s3'] + for variant in after_status['variants']} + check(before == after, 'duplicate delivery rewrote variant objects') + return {'attachment_id': attachment['id'], + 'job_attempts': [job['attempt'] for job in after_status['jobs']], + 'notes': '5 duplicate deliveries were absorbed by the DB fence'} + + +def scenario_reorder(ctx): + compose('stop', 'worker') + try: + attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], + sample_bytes(), 'reorder.png') + ctx['attachment_id'] = attachment['id'] + ctx['tracker'].track(attachment['id']) + flush_outbox(attachment['id']) + status = status_of(attachment['id']) + message = build_message(status, status['jobs'][0]) + stale = dict(message, attempt=99) + publish_raw([stale, message]) + compose('start', 'worker') + wait_ready(ctx['client'], ctx['alice'], attachment['id']) + finally: + compose('start', 'worker') + after_status = status_of(attachment['id']) + skipped = [event for event in after_status['events'] + if event['event'] == 'duplicate_skipped'] + check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages') + check(len([event for event in after_status['events'] + if event['event'] == 'succeeded']) == 1, + 'expected exactly one successful effect') + return {'attachment_id': attachment['id'], + 'job_attempts': [job['attempt'] for job in after_status['jobs']], + 'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'} + + +def crash_scenario(ctx, point, filename, expected_attempt, + expect_lease_expired=False, expect_duplicate=False): + compose('stop', 'worker') + try: + attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], + sample_bytes(), filename) + ctx['attachment_id'] = attachment['id'] + ctx['tracker'].track(attachment['id']) + starts = worker_start_count() + set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash') + compose('start', 'worker') + wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90) + finally: + compose('start', 'worker') + status = status_of(attachment['id']) + job = status['jobs'][0] + check(job['attempt'] == expected_attempt, + 'expected attempt ' + str(expected_attempt) + + ', got ' + str(job['attempt'])) + check(worker_start_count() >= starts + 2, + 'worker process did not restart after the injected crash') + if expect_lease_expired: + check(any(event['event'] == 'lease_expired' + for event in status['events']), + 'missing lease_expired event') + if expect_duplicate: + check(any(event['event'] == 'duplicate_skipped' + and event['detail'].get('reason') == 'already_succeeded' + for event in status['events']), + 'missing already_succeeded duplicate after redelivery') + return {'attachment_id': attachment['id'], + 'job_attempts': [job['attempt']], + 'notes': 'crashed at ' + point} + + +def scenario_crash_before_side_effect(ctx): + return crash_scenario(ctx, 'worker.before_side_effect', + 'crash-before.png', 2, expect_lease_expired=True) + + +def scenario_crash_after_side_effect(ctx): + return crash_scenario(ctx, 'worker.after_side_effect', + 'crash-after.png', 2, expect_lease_expired=True) + + +def scenario_crash_after_commit_before_ack(ctx): + return crash_scenario(ctx, 'worker.after_commit_before_ack', + 'crash-ack.png', 1, expect_duplicate=True) + + +def scenario_crash_after_commit_before_publish(ctx): + before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]} + set_fault('fault:api.after_commit_before_publish', 'crash') + try: + try: + ctx['client'].call( + 'POST', + '/api/v1/chats/' + ctx['chat']['id'] + + '/attachments?filename=crash-publish.png', + (202, 502, 503, 504), + actor=ctx['alice'], + raw=sample_bytes(), + headers={'Content-Type': 'image/png'}, + ) + except Failure: + # A killed replica may close the connection instead of answering. + pass + finally: + clear_fault('fault:api.after_commit_before_publish') + attachment_id = wait_for( + lambda: new_attachment_id(ctx['chat']['id'], before), 30, + 'attachment row after API crash', 1.0, + ) + ctx['attachment_id'] = attachment_id + ctx['tracker'].track(attachment_id) + wait_ready(ctx['client'], ctx['alice'], attachment_id, 90) + status = status_of(attachment_id) + check(len([event for event in status['events'] + if event['event'] == 'created']) == 1, + 'expected exactly one created event') + return {'attachment_id': attachment_id, + 'job_attempts': [job['attempt'] for job in status['jobs']], + 'notes': 'outbox relay published the committed job after API restart'} + + +def scenario_relay_crash_after_publish(ctx): + compose('stop', 'worker') + before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]} + try: + set_fault('fault:relay.after_publish_before_mark', 'crash') + try: + ctx['client'].call( + 'POST', + '/api/v1/chats/' + ctx['chat']['id'] + + '/attachments?filename=relay-crash.png', + (202, 502, 503, 504), + actor=ctx['alice'], + raw=sample_bytes(), + headers={'Content-Type': 'image/png'}, + ) + except Failure: + # The relay may crash the handling replica before the reply flushes. + pass + clear_fault('fault:relay.after_publish_before_mark') + attachment_id = wait_for( + lambda: new_attachment_id(ctx['chat']['id'], before), 30, + 'attachment row after relay crash', 1.0, + ) + ctx['attachment_id'] = attachment_id + ctx['tracker'].track(attachment_id) + wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0, + 60, 'outbox published after relay crash', 1.0) + queue_depth = rabbit_queues().get('attachments.process', {}).get( + 'messages', 0) + compose('start', 'worker') + wait_ready(ctx['client'], ctx['alice'], attachment_id, 90) + finally: + clear_fault('fault:relay.after_publish_before_mark') + compose('start', 'worker') + status = status_of(attachment_id) + check(len([event for event in status['events'] + if event['event'] == 'succeeded']) == 1, + 'expected exactly one succeeded event') + duplicated = any(event['event'] == 'duplicate_skipped' + for event in status['events']) + note = ('relay republished the same message; duplicate absorbed' + if duplicated else + 'relay published once in this run; I1/I2 still hold') + if queue_depth >= 2: + check(duplicated, 'two queued copies but no duplicate_skipped event') + return {'attachment_id': attachment_id, + 'job_attempts': [job['attempt'] for job in status['jobs']], + 'notes': note} + + +def scenario_stale_version(ctx): + check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2, + 'stale_version needs WORKER_CONCURRENCY>=2') + compose('stop', 'worker') + try: + attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], + sample_bytes(), 'stale.png') + attachment_id = attachment['id'] + ctx['attachment_id'] = attachment_id + ctx['tracker'].track(attachment_id) + flush_outbox(attachment_id) + code, payload = cli('reprocess', attachment_id, + '--pipeline-version', '2', '--reason', 'suite') + check(code == 0, 'reprocess v2 failed: ' + str(payload)) + set_fault('fault:worker.after_side_effect:' + attachment_id + ':1', + 'delay:8000') + compose('start', 'worker') + v2_etags = None + + def both_terminal(): + nonlocal v2_etags + status = status_of(attachment_id) + jobs = {(job['pipeline_version']): job for job in status['jobs']} + if '2' in jobs or 2 in jobs: + v2_job = jobs.get(2) + if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None: + v2_etags = {variant['variant']: variant['s3'] + for variant in status['variants'] + if variant['pipeline_version'] == 2} + v1_job = jobs.get(1) + v2_job = jobs.get(2) + return (v1_job and v2_job + and v1_job['status'] == 'superseded' + and v2_job['status'] == 'succeeded') + + wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0) + status = status_of(attachment_id) + check(status['attachment']['ready_pipeline_version'] == 2, + 'ready pipeline version is not 2') + check(all(variant['pipeline_version'] == 2 + for variant in status['variants']), + 'variant rows from v1 survived') + after_etags = {variant['variant']: variant['s3'] + for variant in status['variants'] + if variant['pipeline_version'] == 2} + check(v2_etags == after_etags, 'v1 overwrote v2 variant objects') + + served = ctx['client'].call( + 'GET', + '/api/v1/attachments/' + attachment_id + + '/content?variant=thumbnail', + actor=ctx['alice'], + ) + v2_thumbnail = next(variant for variant in status['variants'] + if variant['variant'] == 'thumbnail') + check(sha256(served['bytes']) == v2_thumbnail['sha256'], + 'served thumbnail is not the v2 object') + check(sha256(served['bytes']) != ctx['baseline']['thumbnail'], + 'served thumbnail equals the v1 baseline') + + v1_job = next(job for job in status['jobs'] + if job['pipeline_version'] == 1) + publish_raw([build_message(status, v1_job)]) + wait_for( + lambda: any(event['event'] == 'duplicate_skipped' + and event['detail'].get('reason') == 'superseded' + for event in status_of(attachment_id)['events']), + 20, 'superseded duplicate skip', 1.0, + ) + finally: + clear_fault('fault:worker.after_side_effect:' + + (attachment['id'] if 'attachment' in locals() else '')) + compose('start', 'worker') + return {'attachment_id': attachment_id, + 'job_attempts': [job['attempt'] for job in status['jobs']], + 'notes': 'v1 finished after v2 and was recorded as superseded'} + + +def scenario_poison(ctx): + before = rabbit_queues().get('attachments.process.dlq', {}).get( + 'messages', 0) + publish_raw(['not-json', '{"schema_version":1}']) + wait_for( + lambda: rabbit_queues().get('attachments.process.dlq', {}).get( + 'messages', 0) >= before + 2, + 15, 'two poison messages in DLQ', 0.5, + ) + return {'attachment_id': None, 'job_attempts': [], + 'notes': 'malformed messages moved to the DLQ without touching jobs'} + + +def scenario_retry_and_replay(ctx): + max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5') + base = int(worker_env('JOB_RETRY_BASE_MS') or '500') + cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000') + compose('stop', 'worker') + try: + attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], + sample_bytes(), 'retry.png') + attachment_id = attachment['id'] + ctx['attachment_id'] = attachment_id + ctx['tracker'].track(attachment_id) + set_fault('fault:worker.before_side_effect:' + attachment_id + ':1', + 'transient') + compose('start', 'worker') + failed = wait_ready(ctx['client'], ctx['alice'], attachment_id, + 90, allow_failed=True) + finally: + compose('start', 'worker') + check(failed['status'] == 'failed', 'transient fault was not exhausted') + check(failed['error_code'] == 'retry_exhausted', + 'expected retry_exhausted, got ' + str(failed['error_code'])) + status = status_of(attachment_id) + retries = [event for event in status['events'] + if event['event'] == 'retry_scheduled'] + check(len(retries) == max_attempts - 1, + 'expected ' + str(max_attempts - 1) + ' retries, got ' + + str(len(retries))) + for index, event in enumerate(retries): + limit = min(cap, base * 2 ** index) + check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit, + 'retry delay exceeds its jitter cap') + ctx['client'].call( + 'GET', '/api/v1/attachments/' + attachment_id + '/content', + 409, actor=ctx['alice'], + ) + code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite') + check(code == 3 and payload.get('reason') in ('no_failed_job', + 'already_ready'), + 'replay of a ready attachment was not refused') + + clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1') + code, payload = cli('replay', attachment_id, '--reason', 'suite') + check(code == 0, 'operator replay failed: ' + str(payload)) + wait_ready(ctx['client'], ctx['alice'], attachment_id, 60) + status = status_of(attachment_id) + replayed = [event for event in status['events'] + if event['event'] == 'replayed'] + check(len(replayed) == 1 and replayed[0]['detail'].get('operator'), + 'missing replayed event or operator audit') + return {'attachment_id': attachment_id, + 'job_attempts': [job['attempt'] for job in status['jobs']], + 'notes': 'transient injected fault exhausted retries; replay recovered'} + + +def scenario_hung_worker(ctx): + timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10') + compose('stop', 'worker') + try: + attachment = upload(ctx['client'], ctx['alice'], ctx['chat'], + sample_bytes(), 'hung.png') + attachment_id = attachment['id'] + ctx['attachment_id'] = attachment_id + ctx['tracker'].track(attachment_id) + set_fault('fault:worker.after_side_effect:' + attachment_id + ':1', + 'delay:' + str((timeout_seconds + 5) * 1000)) + compose('start', 'worker') + wait_ready(ctx['client'], ctx['alice'], attachment_id, 90) + finally: + compose('start', 'worker') + status = status_of(attachment_id) + check(any(event['event'] == 'retry_scheduled' + and event['detail'].get('code') == 'processing_timeout' + for event in status['events']), + 'missing processing_timeout retry') + return {'attachment_id': attachment_id, + 'job_attempts': [job['attempt'] for job in status['jobs']], + 'notes': 'hard timeout aborted the hung render and retried'} + + +def scenario_big_image(ctx): + ctx['compare_baseline'] = False + data = worst_decode_png() + containers = compose('ps', '-q', 'worker').stdout.split() + stop = threading.Event() + peak = {'mb': 0.0} + + def sample(): + while not stop.is_set(): + if containers: + result = docker( + 'stats', '--no-stream', '--format', + '{{.Name}} {{.MemUsage}}', *containers, + check_result=False, + ) + for line in result.stdout.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1]) + if not match: + continue + value = float(match.group(1)) + if match.group(2) == 'GiB': + value *= 1024 + elif match.group(2) == 'KiB': + value /= 1024 + peak['mb'] = max(peak['mb'], value) + stop.wait(0.3) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + try: + def do_upload(index): + client = new_client(ctx['base_url']) + return upload(client, ctx['alice'], ctx['chat'], data, + 'big-' + str(index) + '.png') + + with ThreadPoolExecutor(max_workers=2) as pool: + attachments = list(pool.map(do_upload, (1, 2))) + ctx['attachment_id'] = attachments[0]['id'] + for attachment in attachments: + ctx['tracker'].track(attachment['id']) + results = [wait_ready(ctx['client'], ctx['alice'], + attachment['id'], 120, allow_failed=True) + for attachment in attachments] + finally: + stop.set() + sampler.join(timeout=5) + for meta in results: + check(meta['status'] in ('ready', 'failed'), 'big image stuck') + if meta['status'] == 'failed': + check(meta['error_code'] in ('image_too_large', 'invalid_image'), + 'unexpected big-image error code') + for container in containers: + result = docker('inspect', '-f', '{{.State.OOMKilled}}', container, + check_result=False) + check(result.stdout.strip() == 'false', 'worker was OOM-killed') + return {'attachment_id': attachments[0]['id'], + 'job_attempts': [], + 'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']} + + +SCENARIOS = { + 'duplicate': scenario_duplicate, + 'reorder': scenario_reorder, + 'crash_before_side_effect': scenario_crash_before_side_effect, + 'crash_after_side_effect': scenario_crash_after_side_effect, + 'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack, + 'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish, + 'relay_crash_after_publish': scenario_relay_crash_after_publish, + 'stale_version': scenario_stale_version, + 'poison': scenario_poison, + 'retry_and_replay': scenario_retry_and_replay, + 'hung_worker': scenario_hung_worker, + 'big_image': scenario_big_image, +} + + +def select(name): + if name == 'all': + return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4] + if name == 'level4': + return LEVEL4 + if name == 'track2': + return TRACK2 + if name in SCENARIOS: + return [name] + raise SystemExit('unknown scenario: ' + name) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=None) + parser.add_argument('--ca-file', default=None) + parser.add_argument('--scenario', default='level4') + parser.add_argument('--repeat', type=int, default=1) + parser.add_argument('--output', default=None) + args = parser.parse_args() + + if worker_env('FAULT_INJECTION_ENABLED') != '1': + print('fault injection is disabled; restart the stand with:\n ' + HINT, + file=sys.stderr) + raise SystemExit(2) + + client = new_client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + clear_faults() + reference = upload(client, alice, chat, sample_bytes(), 'reference.png') + ready = wait_ready(client, alice, reference['id'], 60) + baseline = variant_sha(ready) + + selected = select(args.scenario) + results = [] + failed = 0 + for name in selected: + for repeat in range(1, args.repeat + 1): + clear_faults() + tracker = Tracker(args.base_url or 'http://localhost:8080', alice, + args.ca_file) + tracker.start() + started = time.monotonic() + ctx = { + 'client': client, + 'base_url': args.base_url or 'http://localhost:8080', + 'alice': alice, + 'chat': chat, + 'tracker': tracker, + 'baseline': baseline, + 'reference_id': reference['id'], + 'ca_file': args.ca_file, + } + scenario = {'attachment_id': None, 'job_attempts': [], + 'notes': ''} + passed = True + try: + scenario.update(SCENARIOS[name](ctx)) + except Exception as error: # noqa: BLE001 - scenario boundary + passed = False + scenario['notes'] = type(error).__name__ + ': ' + str(error) + finally: + tracker.stop() + clear_faults() + compose('start', 'worker', check_result=False) + attachment_id = scenario['attachment_id'] or ctx.get('attachment_id') + try: + invariants = evaluate( + tracker, attachment_id, baseline, + compare_baseline=ctx.get('compare_baseline', True), + ) + except Exception: # noqa: BLE001 + invariants = {key: False for key in + ('I1', 'I2', 'I3', 'I4', 'I5', 'I6')} + passed = passed and all(invariants.values()) + if not passed: + failed += 1 + result = { + 'name': name, + 'repeat': repeat, + 'passed': passed, + 'duration_s': round(time.monotonic() - started, 3), + 'attachment_id': attachment_id, + 'job_attempts': scenario['job_attempts'], + 'events_summary': scenario.get('events_summary', {}), + 'invariants': invariants, + 'notes': scenario['notes'], + } + results.append(result) + print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) + + ' ' + str(result['duration_s']) + 's ' + + json.dumps(invariants) + ' ' + result['notes'], + flush=True) + + if args.output: + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n') + print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed') + raise SystemExit(1 if failed else 0) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab5/stand.py b/scripts/lab5/stand.py new file mode 100644 index 0000000..387ddff --- /dev/null +++ b/scripts/lab5/stand.py @@ -0,0 +1,180 @@ +"""Shared stdlib-only helpers for the lab5 demo and fault suite.""" +import hashlib +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) + +from client import Client, Failure, check # noqa: E402 + +ENV_FILE = '.env' +COMPOSE = ['docker', 'compose', '--env-file', ENV_FILE] + + +def compose(*args, stdin=None, check_result=True): + result = subprocess.run( + COMPOSE + list(args), + cwd=ROOT, + input=stdin, + text=True, + capture_output=True, + ) + if check_result and result.returncode != 0: + detail = (result.stderr or result.stdout).strip().splitlines() + raise Failure('compose ' + ' '.join(args) + ' failed: ' + + (detail[-1] if detail else 'unknown error')) + return result + + +def docker(*args, check_result=True): + result = subprocess.run( + ['docker'] + list(args), cwd=ROOT, text=True, capture_output=True, + ) + if check_result and result.returncode != 0: + detail = (result.stderr or result.stdout).strip().splitlines() + raise Failure('docker ' + ' '.join(args) + ' failed: ' + + (detail[-1] if detail else 'unknown error')) + return result + + +def cli(*args, stdin=None): + """Run jobs.cli in the API container; returns (exit_code, parsed_json).""" + for _ in range(5): + result = compose( + 'exec', '-T', 'api', 'node', 'dist/jobs/jobs.cli.js', *args, + stdin=stdin, check_result=False, + ) + text = result.stdout.strip() + if text: + try: + return result.returncode, json.loads(text) + except ValueError: + pass + time.sleep(1) + raise Failure('jobs.cli did not return JSON for ' + ' '.join(args)) + + +def set_fault(key, value): + compose('exec', '-T', 'redis', 'redis-cli', 'SET', key, value) + + +def clear_fault(key): + compose('exec', '-T', 'redis', 'redis-cli', 'DEL', key) + + +def clear_faults(): + script = ( + "local keys=redis.call('keys',ARGV[1]) " + "for i=1,#keys do redis.call('del',keys[i]) end return #keys" + ) + compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', script, '0', 'fault:*') + + +def rabbit_queues(): + result = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', + '-p', 'messenger', 'list_queues', + 'name', 'messages', 'type') + queues = {} + for line in result.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] not in ('name', 'Timeout:'): + try: + queues[parts[0]] = {'messages': int(parts[1]), + 'type': parts[2] if len(parts) > 2 else ''} + except ValueError: + continue + return queues + + +def wait_for(predicate, timeout, description='condition', interval=0.5): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = predicate() + if last: + return last + time.sleep(interval) + raise Failure('timed out waiting for ' + description) + + +def upload(client, actor, chat, data, filename='sample.png', + content_type='image/png'): + result = client.call( + 'POST', + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, + 202, + actor=actor, + raw=data, + headers={'Content-Type': content_type}, + schema='Attachment', + ) + check(result['json']['status'] == 'queued', 'upload did not answer queued') + return result['json'] + + +def metadata(client, actor, attachment_id): + return client.call('GET', '/api/v1/attachments/' + attachment_id, + actor=actor, schema='Attachment')['json'] + + +def wait_ready(client, actor, attachment_id, timeout=60, + allow_failed=False): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + meta = metadata(client, actor, attachment_id) + if meta['status'] == 'failed': + if allow_failed: + return meta + raise Failure('attachment failed: ' + + str(meta.get('error_code'))) + if meta['status'] == 'ready': + return meta + time.sleep(0.2) + raise Failure('attachment did not become ready in ' + str(timeout) + 's') + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def variant_sha(meta): + return {item['name']: item['sha256'] for item in meta['variants']} + + +def sample_bytes(): + return (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def worker_env(name): + result = compose('exec', '-T', 'worker', 'printenv', name, + check_result=False) + return result.stdout.strip() + + +def worker_start_count(): + result = compose('logs', '--no-color', 'worker', check_result=False) + return result.stdout.count('worker ready:') + + +def worker_memory_peak(containers): + names = [line for line in containers.splitlines() if line.strip()] + if not names: + return 'n/a' + result = subprocess.run( + ['docker', 'stats', '--no-stream', '--format', + '{{.Name}} {{.MemUsage}}'] + names, + text=True, capture_output=True, + ) + return result.stdout.strip() + + +def print_step(number, text): + print('[' + str(number) + '] ' + text, flush=True) + + +def new_client(base_url, ca_file=None): + return Client(base_url, ca_file) diff --git a/scripts/lab5/worker_down_demo.py b/scripts/lab5/worker_down_demo.py new file mode 100755 index 0000000..ab4beea --- /dev/null +++ b/scripts/lab5/worker_down_demo.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Level-3 demo: work accepted while the worker is down survives restarts.""" +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from stand import ( # noqa: E402 + Failure, + check, + cli, + compose, + new_client, + print_step, + rabbit_queues, + sample_bytes, + sha256, + upload, + wait_for, + wait_ready, +) +from client import arguments, run # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + + +def main(): + args = arguments('lab5 worker-down durability demo') + client = new_client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + data = sample_bytes() + + print_step(1, 'compose ps') + print(compose('ps', '--format', + 'table {{.Service}}\t{{.Status}}').stdout.strip()) + + print_step(2, 'stop worker') + compose('stop', 'worker') + + print_step(3, 'upload while worker is down (expect 202 queued)') + attachment = upload(client, alice, chat, data, 'durability.png') + attachment_id = attachment['id'] + print('attachment=' + attachment_id + ' status=' + attachment['status']) + + print_step(4, 'content and thumbnail must answer 409 before ready') + for query in ('', '?variant=thumbnail'): + result = client.call( + 'GET', + '/api/v1/attachments/' + attachment_id + '/content' + query, + 409, + actor=alice, + ) + check(result['status'] == 409, 'content was served before ready') + + print_step(5, 'link the queued attachment to messages') + with_attachment = client.message( + alice, chat, '', attachments=[attachment_id], + )['json'] + plain = client.message(alice, chat, 'plain while worker is down')['json'] + check(with_attachment['attachment_ids'] == [attachment_id], + 'message lost the queued attachment') + check(plain['attachment_ids'] == [], 'plain message gained an attachment') + print('message=' + with_attachment['id']) + + print_step(6, 'wait for outbox publish and a message in the queue') + def published(): + _code, status = cli('status', attachment_id) + queues = rabbit_queues() + return ( + status['outbox']['pending'] == 0 + and queues.get('attachments.process', {}).get('messages', 0) >= 1 + ) + wait_for(published, 30, 'outbox publish and queue depth >= 1') + queues = rabbit_queues() + print('queue=' + str(queues.get('attachments.process'))) + + print_step(7, 'restart broker and API with volumes') + compose('restart', 'broker', 'api') + compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api') + + print_step(8, 'the quorum message survived the broker restart') + def survived(): + depth = rabbit_queues().get('attachments.process', {}).get('messages', 0) + return depth >= 1 + wait_for(survived, 60, 'queue depth >= 1 after broker restart') + print('queue=' + str(rabbit_queues().get('attachments.process'))) + + print_step(9, 'start worker') + compose('start', 'worker') + + print_step(10, 'wait ready and verify the thumbnail') + ready = wait_ready(client, alice, attachment_id, 60) + thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] + content = client.call( + 'GET', + '/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail', + actor=alice, + ) + check(jpeg_size(content['bytes']) == (256, 160), + 'thumbnail is not 256x160 JPEG') + check(len(content['bytes']) == thumbnail['size_bytes'], + 'thumbnail size differs from metadata') + check(sha256(content['bytes']) == thumbnail['sha256'], + 'thumbnail checksum differs from metadata') + history = client.call( + 'GET', + '/api/v1/chats/' + chat['id'] + '/messages?limit=100', + actor=alice, + schema='MessagePage', + )['json'] + check( + any(attachment_id in item['attachment_ids'] + for item in history['items']), + 'message history lost the attachment', + ) + print('ready attachment=' + attachment_id + + ' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...') + + print_step(11, 'compose ps') + print(compose('ps', '--format', + 'table {{.Service}}\t{{.Status}}').stdout.strip()) + print('PASS: worker-down durability demo') + + +if __name__ == '__main__': + run(main)