From a4d58b2eec0f331fa8165589e4b17a30d6f4b154 Mon Sep 17 00:00:00 2001 From: EgorAnti Date: Thu, 19 Nov 2026 10:57:39 +0300 Subject: [PATCH] 5 --- .env.example | 3 + REPORT.md | 179 ++++++++-- compose.yaml | 3 + evidence/failover-each-2026-11-19.json | 24 ++ evidence/lab5-level5-2026-11-19.md | 213 ++++++++++++ evidence/rolling-update-2026-11-19.json | 23 ++ evidence/upload-memory-2026-11-19.json | 14 + scripts/promote-pipeline.ps1 | 73 ++++ scripts/promote-pipeline.sh | 59 ++++ scripts/replay-failed.ps1 | 1 + scripts/replay-failed.sh | 1 + .../mtusi_messenger/model/AttachmentJob.java | 11 +- .../queue/ExpiredLeaseReaper.java | 6 +- .../repository/AttachmentRepository.java | 14 +- .../PostgresAttachmentRepository.java | 122 ++++--- .../service/AttachmentService.java | 11 +- .../service/ReadinessService.java | 10 +- .../worker/AttachmentWorker.java | 51 ++- .../worker/ImageVariantProcessor.java | 28 +- src/main/resources/application.yml | 3 + ...7__versioned_pipeline_and_fault_points.sql | 33 ++ .../service/AttachmentServiceTest.java | 4 +- .../worker/ImageVariantProcessorTest.java | 28 +- student_tests/README.md | 7 +- student_tests/concurrent_upload_memory.py | 2 +- student_tests/lab5_pipeline_idempotency.py | 314 ++++++++++++++++++ 26 files changed, 1141 insertions(+), 96 deletions(-) create mode 100644 evidence/failover-each-2026-11-19.json create mode 100644 evidence/lab5-level5-2026-11-19.md create mode 100644 evidence/rolling-update-2026-11-19.json create mode 100644 evidence/upload-memory-2026-11-19.json create mode 100644 scripts/promote-pipeline.ps1 create mode 100644 scripts/promote-pipeline.sh create mode 100644 src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql create mode 100644 student_tests/lab5_pipeline_idempotency.py diff --git a/.env.example b/.env.example index c13c3ba8..69c730c5 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,7 @@ ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1 ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1 ATTACHMENT_DLQ_QUEUE=attachments.failed.v1 ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1 +ATTACHMENT_PIPELINE_VERSION=2 WORKER_CONCURRENCY=2 WORKER_PREFETCH=1 JOB_MAX_ATTEMPTS=5 @@ -28,6 +29,8 @@ WORKER_MEMORY_WAIT=2s # Fault injection is opt-in and intended only for an isolated test stand. FAULT_CRASH_AFTER_OUTBOX_COMMIT=false +FAULT_CRASH_BEFORE_SIDE_EFFECT=false +FAULT_CRASH_AFTER_SIDE_EFFECT=false FAULT_CRASH_AFTER_RESULT_COMMIT=false S3_REGION=us-east-1 diff --git a/REPORT.md b/REPORT.md index 687415e4..6afcfe27 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,10 +1,10 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка: 4 +- Автор, группа: Антипин Егор Иванович +- Целевая оценка: **5**, трек 2 «Доказанная идемпотентность pipeline» - Commit/PR: TODO после публикации - Библиотеки: Spring AMQP, RabbitMQ Java client, Thumbnailator, webp-imageio, MinIO SDK, PostgreSQL/Flyway -- Помощь ИИ: проектирование pipeline, реализация и подготовка тестов; результат проверен сборкой, unit- и black-box-тестами +- Помощь ИИ: проектирование versioned pipeline, fencing и fault-injection, реализация, подготовка тестов и черновиков evidence; результат проверен сборкой, unit-, black-box- и аварийными тестами ## Запуск из чистого clone @@ -18,24 +18,135 @@ docker compose --env-file .env up -d --build python -X utf8 tests/smoke.py --base-url http://localhost:8080 ``` -`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных выполняется только явной командой оператора с `--volumes`. +Автоматические `student_tests/lab5_*.py` используют `.env.example` и рассчитаны +на одноразовый учебный стенд; для них удобно поднимать Compose с тем же файлом: + +```sh +docker compose --env-file .env.example up -d --build +python -X utf8 student_tests/lab5_pipeline_idempotency.py +``` + +`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных +выполняется только явной командой оператора с `--volumes`. ## Pipeline и гарантии -Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет неизменённый original в приватный `staging//original`, затем одной транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`. Запись job является transactional outbox. Фоновый publisher публикует persistent JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а HTTP `202` означает, что staging и намерение обработки уже устойчивы. +Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет +неизменённый original в приватный `staging//original`, затем одной +транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`. +Запись job является transactional outbox. Фоновый publisher публикует persistent +JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job +опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а +HTTP `202` означает, что staging и намерение обработки уже устойчивы. -Отдельный `worker` захватывает job с lease, читает доверенный staging key из PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только `schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют. Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже `ready` job — no-op. +Отдельный `worker` захватывает job с lease, читает доверенный staging key из +PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только +`schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, +`created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют. +Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant +rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже +`ready` job — no-op. -Обязательные варианты pipeline version 1: +Обязательные варианты для активной версии: | Вариант | Правило | Формат/качество | | --- | --- | --- | | `original` | bytes и SHA-256 совпадают с upload | исходный MIME | -| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality 0.85 | -| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality 0.88 | -| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality 0.82 | +| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality зависит от версии | +| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality зависит от версии | +| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality зависит от версии | -Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных metadata. +Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных +metadata. + +## Versioned pipeline v1/v2 + +Миграция `V7__versioned_pipeline_and_fault_points.sql` добавляет: + +- `attachments.pipeline_version`; +- уникальность `attachment_jobs(attachment_id, pipeline_version)`; +- состояние `superseded`; +- audit-таблицу `attachment_pipeline_promotions`; +- маркеры fault-injection до и после side effect. + +`ImageVariantProcessor` выбирает профиль по `pipeline_version`: v1 и v2 имеют +разные quality для JPEG/WebP, поэтому результаты версий различаются побайтово. +Ключи объектов версионированы: `thumbnail-v1.jpg`, `thumbnail-v2.jpg`, +`optimized-v1.webp` и т.д.; общий `original` неизменяем. Старая версия физически +не может перезаписать объект новой версии. Одна и та же `pipeline_version` +обязана быть детерминированной: изменение quality/алгоритма оформляется новой +версией, иначе `ON CONFLICT DO NOTHING` сохранит уже записанные variant rows. + +Повышение версии — одна аудированная транзакция +`scripts/promote-pipeline.ps1`/`.sh`: + +1. блокирует attachment; +2. переводит старые `queued/processing` jobs в `superseded` и `published=true`; +3. создаёт новую job версии `N`; +4. обновляет `attachments.pipeline_version` и `status=queued`; +5. пишет `attachment_pipeline_promotions`. + +Fencing выполняется в каждом переходе состояния: `claim`, `markReady`, +`markFailed`, retry, expired-lease recovery и replay проверяют одновременно +`job.pipeline_version`, `attachments.pipeline_version`, lease token и terminal +state. Поэтому старая job после promotion не может ни получить lease, ни +записать variants, ни перевести attachment в `ready`. + +Если v1 уже удалил staging после безопасного COMMIT, worker версии v2 читает +неизменяемый `attachments//original`; содержимое и SHA-256 те же. + +## Fault injection и инварианты + +| Точка | Поведение | +| --- | --- | +| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API падает после COMMIT attachment/outbox, до publish; republisher восстанавливает намерение | +| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker падает до S3 PUT; lease истекает, reaper возвращает job в outbox | +| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker падает после детерминированных S3 PUT, до result COMMIT; повтор сходится к тем же объектам | +| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker падает после atomic `ready` COMMIT, до consumer ack; redelivery — no-op | + +Все флаги по умолчанию выключены и предназначены только для изолированного +стенда. Автоматический `student_tests/lab5_pipeline_idempotency.py` проверяет: + +- duplicate: две одинаковые v2-доставки дают один результат; +- reorder: v2 доставляется раньше stale v1; +- stale: v1 после v2 не меняет snapshot активного результата; +- superseded: повышение ещё queued v1 job отменяет её без variant rows; +- crash до S3 side effect: 0 объектов, 0 result rows, восстановление через + повторную попытку; +- crash после S3 side effect: 4 объекта до COMMIT, после повтора — те же 4 + объекта и ровно 3 variant rows; +- crash после result COMMIT: redelivery не дублирует variants; +- все 5 подтверждённых вложений остаются `ready`. + +Зафиксированные инварианты: + +1. `ready` активной версии ⇒ ровно три variant rows активной версии и нет + `queued/processing` jobs. +2. Не более одной активной job на attachment. +3. `(attachment_id, pipeline_version, name)` уникален; повтор не создаёт дубль. +4. Versioned object keys не позволяют старой версии перезаписать новую. +5. `pipeline_version` повышается только аудированной транзакцией; старая + `queued/processing` job становится `superseded`. +6. Подтверждённые задачи не теряются после duplicate/reorder/stale/crash. +7. Нет variant rows версии больше активной. + +## Почему это не exactly-once доставка + +RabbitMQ обеспечивает **at-least-once**: сообщение может быть доставлено +повторно после потери соединения или падения worker до ack; порядок сообщений не +является бизнес-порядком. Publisher confirm подтверждает только приём брокером, +а consumer ack идёт после PostgreSQL COMMIT. Между S3 PUT и COMMIT нет +распределённой транзакции, поэтому crash в этом окне оставляет уже выполненные +side effects и вызывает повторную обработку. + +Идемпотентность достигнута на уровне **эффекта**, а не доставки: deterministic +object keys, `ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady` +приводят любые повторы к одному наблюдаемому состоянию активной версии. То же +самое можно назвать **effectively-once** результатом при at-least-once +доставке. Exactly-once доставка потребовала бы транзакционной связи RabbitMQ, +PostgreSQL и S3 или эквивалентного распределённого протокола; в этой учебной +архитектуре её нет, и тест сознательно демонстрирует лишние повторы S3 PUT, +которые не меняют результат. ## Ограничения worker и retry @@ -47,25 +158,49 @@ Upload разделён на durable-границы: API проверяет byte - Не более `JOB_MAX_ATTEMPTS=5`. Временные ошибки получают exponential backoff с jitter 50–100% и cap 30s. Ошибки decode/format/size постоянные и сразу становятся безопасным `failed/error_code`; exhausted retry — `retry_exhausted`. - Permanent malformed broker delivery и исчерпанная работа dead-lettered в durable `attachments.failed.v1`. Реестр failed остаётся в PostgreSQL. -Операторский replay выполняется `scripts/replay-failed.ps1 -JobId ` или `scripts/replay-failed.sh `. Одна транзакция блокирует одновременно failed job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние. Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`. +Операторский replay выполняется `scripts/replay-failed.ps1 -JobId ` или +`scripts/replay-failed.sh `. Одна транзакция блокирует одновременно failed +job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние. +Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`; replay +также требует совпадения `pipeline_version`. -После безопасного COMMIT результата worker best-effort удаляет staging. `StagingCleaner` удаляет только неактивные объекты старше `STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён. +После безопасного COMMIT результата worker best-effort удаляет staging. +`StagingCleaner` удаляет только неактивные объекты старше +`STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён. Отдельного +GC для orphan-объектов старых версий после promotion «в полёте» нет: active +результат они не перезаписывают, но место могут занимать. -## Fault injection и проверки +## Проверки | Критерий | Команда | Результат | | --- | --- | --- | -| Unit tests | `gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' --tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'` | `BUILD SUCCESSFUL` | -| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS | +| Unit + integration | `gradlew test` | `BUILD SUCCESSFUL`, 30 tests | +| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS, exit 0 | | Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:8080` | PASS | -| Worker stop, broker/API restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows | -| COMMIT→publish и result COMMIT→ack crash | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи автоматически дошли до ready | +| Трек 5.2 | `python -X utf8 student_tests/lab5_pipeline_idempotency.py` | PASS: version fencing, superseded, duplicate/reorder, pre/post-side-effect crashes | +| Worker stop/restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows | +| COMMIT→publish и result COMMIT→ack | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи дошли до ready | | DLQ, replay, lease | `python -X utf8 student_tests/lab5_dlq_replay_lease.py` | PASS; DLQ +1, одна audit-запись, ready не replayed, expired lease восстановлен | +| Lab3: shared idempotency | `python -X utf8 student_tests/idempotency_concurrent.py` | PASS; 32 запроса, 2 реплики, одно сообщение | +| Lab3: readiness gate | `python -X utf8 student_tests/readiness_gate.py` | PASS; обе реплики 503 на сломанной схеме и восстановились | +| Lab3: failover каждой реплики | `python -X utf8 student_tests/failover_each.py ...` | PASS; 2 реплики, confirmed writes сохранены | +| Lab3: rolling update | `python -X utf8 student_tests/rolling_update.py ...` | PASS; 4 instance ID, 335 успешных записей, 3 переходных отказа | +| Lab4: negative | `python -X utf8 student_tests/lab4_negative.py` | PASS; ownership/MIME/corruption/limit/filename | +| Lab4: upload memory | `python -X utf8 student_tests/concurrent_upload_memory.py ...` | PASS; 12/12 uploads, 2 API реплики | -Fault flags выключены по умолчанию. `FAULT_CRASH_AFTER_OUTBOX_COMMIT=true` один раз завершает API через `Runtime.halt(87)` после DB COMMIT до publish. После restart republisher находит `published=false`, и job достигает ready. `FAULT_CRASH_AFTER_RESULT_COMMIT=true` один раз завершает worker через `Runtime.halt(86)` после атомарного ready до consumer ack; RabbitMQ redelivery проверяет ready и завершает ту же job без второго набора вариантов. - -Подробный журнал: `evidence/lab5-level4-2026-11-19.md`. +Подробный журнал с SQL-инвариантами и JSON-агрегатами: +`evidence/lab5-level5-2026-11-19.md`. Предыдущие уровни: +`evidence/lab5-level4-2026-11-19.md`, `evidence/lab4-2026-11-11.md`. ## Ограничения -Это уровень 4, не исследовательский трек 5: производительность для матрицы concurrency/prefetch не измерялась, multi-version reorder не исследовался. RabbitMQ, PostgreSQL и MinIO развёрнуты по одному экземпляру; volumes переживают container restart, но не потерю host/disk. At-least-once + идемпотентный эффект не является exactly-once доставкой. +- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO; + эксперимент не доказывает отказоустойчивость к потере host/диска. +- Exactly-once доставки нет; доказан at-least-once + идемпотентный эффект. +- Versioned orphan-объекты старых версий после promotion «в полёте» не + перезаписывают active результат, но автоматического GC для них нет. +- `promote-pipeline` — операторский скрипт с SQL-транзакцией, не публичный API. +- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант; + на чистом volume и для новых данных действует инвариант трёх вариантов. +- Числа failover/rolling — проверка сохранности на одном Windows-хосте, не + benchmark пропускной способности. diff --git a/compose.yaml b/compose.yaml index 82fa8d81..6ccc63cb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -41,6 +41,7 @@ x-api-environment: &api-environment ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1} ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1} ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1} + ATTACHMENT_PIPELINE_VERSION: ${ATTACHMENT_PIPELINE_VERSION:-2} WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} WORKER_PREFETCH: ${WORKER_PREFETCH:-1} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} @@ -258,6 +259,8 @@ services: MESSENGER_ROLE: worker SPRING_MAIN_WEB_APPLICATION_TYPE: none JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70 + FAULT_CRASH_BEFORE_SIDE_EFFECT: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:-false} + FAULT_CRASH_AFTER_SIDE_EFFECT: ${FAULT_CRASH_AFTER_SIDE_EFFECT:-false} FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false} depends_on: migrate: diff --git a/evidence/failover-each-2026-11-19.json b/evidence/failover-each-2026-11-19.json new file mode 100644 index 00000000..a8241bd9 --- /dev/null +++ b/evidence/failover-each-2026-11-19.json @@ -0,0 +1,24 @@ +{ + "scenario": "sequential failover of each API replica", + "replicas": 2, + "results": [ + { + "target_instance_id": "8e081627dfc3", + "window_seconds": 15.775, + "successful_reads": 3, + "successful_writes": 4, + "failed_attempts": 0, + "successful_rps": 0.444, + "first_recovery_seconds": 5.018 + }, + { + "target_instance_id": "95b5db854bee", + "window_seconds": 16.774, + "successful_reads": 3, + "successful_writes": 3, + "failed_attempts": 1, + "successful_rps": 0.358, + "first_recovery_seconds": 6.15 + } + ] +} diff --git a/evidence/lab5-level5-2026-11-19.md b/evidence/lab5-level5-2026-11-19.md new file mode 100644 index 00000000..438c45bb --- /dev/null +++ b/evidence/lab5-level5-2026-11-19.md @@ -0,0 +1,213 @@ +# Lab5, трек 5.2 — доказанная идемпотентность pipeline + +Дата: 2026-11-19. Исходная точка: `75d2f78` + рабочее дерево с изменениями lab5. +Стенд: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23 (контейнеры и Gradle +toolchain), Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, Redis 7.4, +MinIO `bitnamilegacy/minio:2025.4.22-debian-12-r1`. + +Все пользователи, сообщения и изображения синтетические. Секреты, session tokens, +содержимое сообщений и signed URL в evidence не сохранялись. + +## Гипотеза и инварианты + +Гипотеза: при versioned pipeline и fencing по `(attachment_id, pipeline_version, +lease)` повторная доставка, перестановка сообщений, crash в окнах до/после side +effect и устаревшая job не теряют подтверждённую работу и не заменяют результат +новой версии старым. + +Проверяемые инварианты: + +1. `attachments.status='ready'` для активной версии означает ровно три variant + rows этой версии (`thumbnail`, `crop`, `optimized`) и отсутствие + `queued/processing` jobs у вложения. +2. Для вложения существует не более одной активной (`queued/processing`) job. +3. Ключ variant rows — `(attachment_id, pipeline_version, name)`; повторная + запись не создаёт дубль. +4. Объектные ключи версионированы: `thumbnail-vN.jpg`, `crop-vN.jpg`, + `optimized-vN.webp`; `original` общий и неизменяемый. Поэтому v1 не может + перезаписать v2. +5. `attachments.pipeline_version` монотонно повышается только аудированной + транзакцией `promote-pipeline`; старая `queued/processing` job получает + `superseded` и `published=true`. +6. Все подтверждённые тестом задачи (`status='ready'`) остаются `ready` после + duplicate/reorder/stale доставок. +7. В active version нет variant rows с `pipeline_version` больше + `attachments.pipeline_version`. +8. Одна и та же `pipeline_version` детерминирована; изменение профиля + оформляется новой версией, потому что `ON CONFLICT DO NOTHING` не перезаписывает + уже зафиксированные variant rows. + +## Команды и результаты + +```text +python -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +docker compose --env-file .env.example config --quiet +exit 0 + +.\gradlew.bat test +BUILD SUCCESSFUL (30 tests; full suite against live Compose PostgreSQL/Redis/MinIO) +``` + +Для полного Gradle suite хранилища Compose не публикуют порты наружу; на время +прогона использовались временные localhost-туннели к `postgres:5432`, +`redis:6379`, `rabbitmq:5672`, `minio:9000`. После прогона туннели удалены. + +### Основной эксперимент трека + +```text +python -X utf8 student_tests/lab5_pipeline_idempotency.py +PASS: version fencing, superseded, duplicate/reorder and pre/post-side-effect crashes +``` + +Тест автоматически: + +1. создаёт v1 attachment, дожидается `ready` и снимает snapshot трёх variant rows; +2. останавливает worker, повышает версию до v2 через `scripts/promote-pipeline`, + публикует `v2, v2, v1` (порядок доставки не совпадает с порядком создания) и + запускает worker; +3. проверяет, что v2 дал отдельный результат, в БД ровно 6 rows (3 v1 + 3 v2), + active version = 2, а повторные stale v1 / duplicate v2 не меняют snapshot; +4. отдельно повышает ещё **queued** v1 job: она становится + `superseded|published=true|superseded_by_v2`, v1 variant rows не появляются, + v2 доходит до `ready` с тремя вариантами; +5. вставляет crash до S3 side effect, после S3 side effect и после result + COMMIT; каждый раз проверяет восстановление через lease/reaper/redelivery; +6. проверяет, что все 5 подтверждённых вложений остались `ready`. + +### Наблюдаемое состояние после прогона + +Пример versioned-attachment (сокращённые ID приведены для читаемости): + +```text +attachment A (v1 -> v2, сначала завершён v1) + active=2 ready + job v1 ready published=t attempt=1 + job v2 ready published=t attempt=1 + variant keys: crop-v1, optimized-v1, thumbnail-v1, + crop-v2, optimized-v2, thumbnail-v2 + +attachment B (queued v1 повышен до v2) + active=2 ready + job v1 superseded published=t last_error=superseded_by_v2 + job v2 ready published=t attempt=1 + variant keys: crop-v2, optimized-v2, thumbnail-v2 (v1 rows отсутствуют) +``` + +SQL-инварианты по всем attachment, созданным 2026-11-19 в ходе финального +прогона и регрессии lab5: + +```text +superseded_jobs|1 +today_promotions|9 +today_ready|55 +today_ready_bad_variants|0 +ready_with_nonterminal_job|0 +attachments_with_two_active_jobs|0 +duplicate_variant_keys|0 +stale_results_for_active_version|0 +``` + +Полный volume содержит также 3 attachment, созданных 2026-11-18 до миграции V7 +(тогда уровень 3 сохранял только original + thumbnail). Это исторические данные +предыдущего стенда; они не являются результатом текущего versioned pipeline. +Запросы инвариантов выше ограничены `created_at >= '2026-11-19'`; на чистом +volume исторических строк нет. + +### Точки отказа + +| Fault | Где срабатывает | Наблюдение | +| --- | --- | --- | +| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker, до чтения staging/S3 PUT, `halt(84)` | 0 variant rows, 0 объектов; reaper возвращает job в `queued`, attempt увеличивается, задача доходит до `ready` | +| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker, после 4 S3 PUT, до result COMMIT, `halt(85)` | objects=4, variant rows=0; повтор пишет те же deterministic keys и коммитит результат, objects остаётся 4 | +| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker, после atomic `ready` COMMIT, до ack, `halt(86)` | redelivery видит `ready` и завершает no-op; 3 variant rows не дублируются | +| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API, после COMMIT attachment/outbox, до publish, `halt(87)` | republisher находит `published=false` и публикует сохранённое намерение; задача доходит до `ready` | + +Crash-точки включаются только переменными окружения и по умолчанию выключены; +они предназначены для изолированного учебного стенда. + +## Регрессия уровней 3–4 + +```text +python -X utf8 tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +python -X utf8 student_tests/lab5_worker_recovery.py +PASS: stopped worker, durable restart recovery and duplicate delivery + +python -X utf8 student_tests/lab5_fault_injection.py +PASS: outbox commit/publish crash and result commit/ack crash recovered + +python -X utf8 student_tests/lab5_dlq_replay_lease.py +PASS: DLQ, guarded replay and expired lease recovery + +python -X utf8 student_tests/idempotency_concurrent.py +PASS: 32 concurrent requests used 2 replicas and created one message + +python -X utf8 student_tests/readiness_gate.py +PASS: 2 unhealthy API containers refused business traffic with 503 and recovered after schema restoration + +python -X utf8 student_tests/failover_each.py --action-script scripts/stop-one.cmd \ + --restore-script scripts/restore-api.cmd --window 15 \ + --output evidence/failover-each-2026-11-19.json +(JSON: 2 replicas sequentially stopped and restored; confirmed writes preserved) + +python -X utf8 student_tests/rolling_update.py --action-script scripts/rolling-update.cmd \ + --duration 180 --concurrency 4 \ + --output evidence/rolling-update-2026-11-19.json +(JSON: 4 observed replacement instance IDs; 335 successful writes, 3 transitional failures) + +python -X utf8 student_tests/lab4_negative.py +PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks + +python -X utf8 student_tests/concurrent_upload_memory.py \ + --output evidence/upload-memory-2026-11-19.json +PASS: concurrent upload memory experiment (12/12 uploads accepted, 2 API replicas) +``` + +`readiness_gate.py` проверяет Google-подобный gate по обязательной схеме: +`ReadinessService` теперь пробует обе таблицы — `message_idempotency` (lab3) и +`attachments` (lab4/lab5). `concurrent_upload_memory.py` обновлён с lab4-контракта +`201` на lab5-контракт `202`. + +## Почему это at-least-once, а не exactly-once + +RabbitMQ даёт at-least-once: сообщение может быть доставлено повторно после +потери соединения или падения worker до ack; порядок между разными сообщениями +не является бизнес-порядком. Поэтому duplicate и reorder — нормальные состояния, +а не аномалии. + +Наш pipeline делает **эффект** идемпотентным, но не делает доставку exactly-once: + +- publisher confirm подтверждает только приём сообщения брокером, а consumer ack + отправляется после PostgreSQL COMMIT; между S3 PUT и COMMIT нет распределённой + транзакции; +- crash после S3 PUT и до COMMIT оставляет объекты, а crash после COMMIT и до ack + вызывает redelivery; оба окна специально инжектируются тестом; +- повторная обработка может повторно выполнить S3 PUT, но deterministic keys, + `ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady` не создают + второго набора результатов и не меняют active version; +- stale job ограничена `pipeline_version` и lease в каждом переходе состояния, + поэтому не может записать результат поверх новой версии. + +Корректная формулировка: **at-least-once delivery + idempotent effect = +effectively-once наблюдаемый результат для активной версии**. Exactly-once +доставка потребовала бы транзакционной связи RabbitMQ, PostgreSQL и S3 (или +эквивалентного распределённого протокола), которой в этой учебной архитектуре +нет и которая не нужна для данных инвариантов. + +## Границы + +- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO; + эксперимент не доказывает отказоустойчивость к потере host/диска. +- Versioned S3 objects старой версии после promotion «в полёте» могут остаться + orphan-объектами: active result они не перезаписывают, но отдельного GC для + versioned objects нет (чистится только staging). +- `superseded`/promotion — операторская SQL-транзакция через скрипт, не публичный + HTTP API. +- Числа failover/rolling в JSON — проверка сохранности и восстановления на одном + Windows-хосте, не benchmark пропускной способности; latency включает + особенность localhost dual-stack в Docker Desktop. +- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант; + на чистом volume и для новых данных действует инвариант трёх вариантов. diff --git a/evidence/rolling-update-2026-11-19.json b/evidence/rolling-update-2026-11-19.json new file mode 100644 index 00000000..6362bc49 --- /dev/null +++ b/evidence/rolling-update-2026-11-19.json @@ -0,0 +1,23 @@ +{ + "scenario": "sequential API replacement under closed-loop writes", + "requested_duration_seconds": 180.0, + "elapsed_seconds": 182.022, + "rolling_action_seconds": 125.275, + "concurrency": 4, + "observed_instance_ids": [ + "431acab3a50c", + "8e081627dfc3", + "d67820ebd09d", + "fdff9a67741c" + ], + "successful_requests": 335, + "failed_requests": 3, + "error_ratio": 0.008876, + "successful_rps": 1.84, + "successful_latency_ms": { + "p50": 2092.164, + "p95": 2252.301, + "p99": 3064.667 + }, + "note": "Closed-loop result on one host; success latency excludes failures." +} diff --git a/evidence/upload-memory-2026-11-19.json b/evidence/upload-memory-2026-11-19.json new file mode 100644 index 00000000..252469de --- /dev/null +++ b/evidence/upload-memory-2026-11-19.json @@ -0,0 +1,14 @@ +{ + "concurrency": 12, + "successful_uploads": 12, + "payload_bytes_each": 6835833, + "duration_seconds": 31.241, + "api_replicas": 2, + "memory_samples": 10, + "baseline_api_memory_bytes": 665321471, + "peak_api_memory_bytes": 833198489, + "peak_increase_bytes": 167877018, + "configured_max_concurrent_uploads_per_replica": 3, + "configured_max_upload_bytes": 10485760, + "configured_max_image_pixels": 40000000 +} diff --git a/scripts/promote-pipeline.ps1 b/scripts/promote-pipeline.ps1 new file mode 100644 index 00000000..6eae43de --- /dev/null +++ b/scripts/promote-pipeline.ps1 @@ -0,0 +1,73 @@ +param( + [Parameter(Mandatory = $true)] + [string]$AttachmentId, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2)] + [int]$PipelineVersion, + [string]$RequestedBy = $env:USERNAME, + [string]$EnvFile = ".env" +) + +$parsedAttachmentId = [guid]::Empty +if (-not [guid]::TryParse($AttachmentId, [ref]$parsedAttachmentId)) { + throw "AttachmentId must be a UUID" +} +if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') { + throw "RequestedBy contains unsupported characters" +} + +$sql = @" +WITH candidate AS ( + SELECT id, pipeline_version + FROM attachments + WHERE id = '$parsedAttachmentId'::uuid AND pipeline_version < $PipelineVersion + FOR UPDATE +), superseded AS ( + UPDATE attachment_jobs j + SET status = 'superseded', published = true, + last_error_code = 'superseded_by_v$PipelineVersion', + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c + WHERE j.attachment_id = c.id AND j.pipeline_version < $PipelineVersion + AND j.status IN ('queued', 'processing') + RETURNING j.id +), created AS ( + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, attempt, status, published, + next_attempt_at, created_at, updated_at) + SELECT gen_random_uuid(), id, $PipelineVersion, 1, 'queued', false, + clock_timestamp(), clock_timestamp(), clock_timestamp() + FROM candidate + ON CONFLICT (attachment_id, pipeline_version) DO NOTHING + RETURNING id, attachment_id +), promoted AS ( + UPDATE attachments a + SET pipeline_version = $PipelineVersion, status = 'queued', error_code = NULL, + updated_at = clock_timestamp() + FROM candidate c, created n + WHERE a.id = c.id AND n.attachment_id = c.id + RETURNING a.id +), audited AS ( + INSERT INTO attachment_pipeline_promotions + (attachment_id, from_version, to_version, job_id, requested_by) + SELECT c.id, c.pipeline_version, $PipelineVersion, n.id, '$RequestedBy' + FROM candidate c + JOIN created n ON n.attachment_id = c.id + JOIN promoted p ON p.id = c.id + RETURNING job_id +) +SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op'); +"@ + +$result = docker compose --env-file $EnvFile exec -T postgres ` + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql +if ($LASTEXITCODE -ne 0) { + throw "Pipeline promotion transaction failed" +} + +$last = ($result | Select-Object -Last 1).Trim() +if ($last -eq "no-op") { + Write-Output "No-op: attachment is missing, already at v$PipelineVersion, or that version already exists" +} else { + Write-Output "Promoted attachment $parsedAttachmentId to pipeline v$PipelineVersion with job $last" +} diff --git a/scripts/promote-pipeline.sh b/scripts/promote-pipeline.sh new file mode 100644 index 00000000..dbed0ac8 --- /dev/null +++ b/scripts/promote-pipeline.sh @@ -0,0 +1,59 @@ +#!/bin/sh +set -eu + +attachment_id="${1:?usage: promote-pipeline.sh ATTACHMENT_UUID PIPELINE_VERSION [REQUESTED_BY] [ENV_FILE]}" +pipeline_version="${2:?pipeline version is required}" +requested_by="${3:-operator}" +env_file="${4:-.env}" + +case "$attachment_id" in + ????????-????-????-????-????????????) ;; + *) echo "AttachmentId must be a UUID" >&2; exit 2 ;; +esac +case "$pipeline_version" in + 1|2) ;; + *) echo "PipelineVersion must be 1 or 2" >&2; exit 2 ;; +esac +case "$requested_by" in + *[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;; +esac + +sql="WITH candidate AS ( + SELECT id, pipeline_version FROM attachments + WHERE id = '$attachment_id'::uuid AND pipeline_version < $pipeline_version FOR UPDATE +), superseded AS ( + UPDATE attachment_jobs j SET status = 'superseded', published = true, + last_error_code = 'superseded_by_v$pipeline_version', + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c WHERE j.attachment_id = c.id + AND j.pipeline_version < $pipeline_version AND j.status IN ('queued', 'processing') + RETURNING j.id +), created AS ( + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, attempt, status, published, + next_attempt_at, created_at, updated_at) + SELECT gen_random_uuid(), id, $pipeline_version, 1, 'queued', false, + clock_timestamp(), clock_timestamp(), clock_timestamp() FROM candidate + ON CONFLICT (attachment_id, pipeline_version) DO NOTHING + RETURNING id, attachment_id +), promoted AS ( + UPDATE attachments a SET pipeline_version = $pipeline_version, + status = 'queued', error_code = NULL, updated_at = clock_timestamp() + FROM candidate c, created n WHERE a.id = c.id AND n.attachment_id = c.id + RETURNING a.id +), audited AS ( + INSERT INTO attachment_pipeline_promotions + (attachment_id, from_version, to_version, job_id, requested_by) + SELECT c.id, c.pipeline_version, $pipeline_version, n.id, '$requested_by' + FROM candidate c JOIN created n ON n.attachment_id = c.id + JOIN promoted p ON p.id = c.id RETURNING job_id +) +SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');" + +result="$(docker compose --env-file "$env_file" exec -T postgres \ + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")" +if [ "$result" = "no-op" ]; then + echo "No-op: attachment is missing, already at v$pipeline_version, or that version already exists" +else + echo "Promoted attachment $attachment_id to pipeline v$pipeline_version with job $result" +fi diff --git a/scripts/replay-failed.ps1 b/scripts/replay-failed.ps1 index 4bd2343e..ea9c4bd5 100644 --- a/scripts/replay-failed.ps1 +++ b/scripts/replay-failed.ps1 @@ -20,6 +20,7 @@ WITH candidate AS ( JOIN attachments a ON a.id = j.attachment_id WHERE j.id = '$parsedJobId'::uuid AND j.status = 'failed' AND a.status = 'failed' + AND j.pipeline_version = a.pipeline_version FOR UPDATE OF j, a ), audited AS ( INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt) diff --git a/scripts/replay-failed.sh b/scripts/replay-failed.sh index 25fa5a1c..4bfe11bc 100644 --- a/scripts/replay-failed.sh +++ b/scripts/replay-failed.sh @@ -17,6 +17,7 @@ sql="WITH candidate AS ( SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed' + AND j.pipeline_version = a.pipeline_version FOR UPDATE OF j, a ), audited AS ( INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt) diff --git a/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java b/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java index 0708e84e..a189403c 100644 --- a/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java +++ b/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java @@ -15,19 +15,22 @@ public record AttachmentJob( @JsonProperty("created_at") Instant createdAt) { public static final int SCHEMA_VERSION = 1; - public static final int PIPELINE_VERSION = 1; + public static final int MIN_PIPELINE_VERSION = 1; + public static final int CURRENT_PIPELINE_VERSION = 2; public static final String TYPE = "attachment.process.v1"; public AttachmentJob { if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type) - || pipelineVersion < 1 || attempt < 1) { + || pipelineVersion < MIN_PIPELINE_VERSION + || pipelineVersion > CURRENT_PIPELINE_VERSION || attempt < 1) { throw new IllegalArgumentException("Unsupported attachment job message"); } } - public static AttachmentJob first(UUID jobId, UUID attachmentId, Instant createdAt) { + public static AttachmentJob first( + UUID jobId, UUID attachmentId, int pipelineVersion, Instant createdAt) { return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId, - PIPELINE_VERSION, 1, createdAt); + pipelineVersion, 1, createdAt); } public AttachmentJob nextAttempt(int nextAttempt) { diff --git a/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java b/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java index 7672b84f..4ddeb98c 100644 --- a/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java +++ b/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java @@ -38,11 +38,13 @@ public class ExpiredLeaseReaper { try { transaction.executeWithoutResult(status -> { if (job.attempt() >= maxAttempts) { - attachments.failExpired(job.jobId(), job.attachmentId(), "retry_exhausted"); + attachments.failExpired(job.jobId(), job.attachmentId(), + job.pipelineVersion(), "retry_exhausted"); } else { int nextAttempt = job.attempt() + 1; attachments.requeueExpired( - job.jobId(), job.attachmentId(), nextAttempt, "processing_timeout", + job.jobId(), job.attachmentId(), job.pipelineVersion(), + nextAttempt, "processing_timeout", clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt))); } }); diff --git a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java index 0e0ad5fc..1253575e 100644 --- a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java +++ b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java @@ -21,16 +21,20 @@ public interface AttachmentRepository { boolean markPublishFaultInjected(UUID jobId); List findUnpublishedJobs(int limit); int currentAttempt(UUID jobId); - boolean claim(UUID jobId, UUID attachmentId, int attempt, UUID leaseToken, Duration leaseDuration); - boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken, + boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, + int attempt, UUID leaseToken, Duration leaseDuration); + boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken); + boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken); + boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, UUID leaseToken, int nextAttempt, String errorCode, Instant nextAttemptAt); CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion, List variants, boolean injectCrashAfterCommit); - boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode); + boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion, + UUID leaseToken, String errorCode); List findExpiredJobs(int limit); - boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt, + boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, int nextAttempt, String errorCode, Instant nextAttemptAt); - boolean failExpired(UUID jobId, UUID attachmentId, String errorCode); + boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, String errorCode); record CompletionResult(boolean completed, boolean crashAfterCommit) { } diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java index 93794691..bcfa5313 100644 --- a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java @@ -35,11 +35,13 @@ public class PostgresAttachmentRepository implements AttachmentRepository { jdbc.update(""" INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, - sha256, object_key, staging_key, status, error_code, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?) + sha256, object_key, staging_key, pipeline_version, + status, error_code, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?) """, attachment.id(), attachment.chatId(), attachment.ownerId(), attachment.filename(), attachment.contentType(), attachment.sizeBytes(), attachment.sha256(), attachment.objectKey(), attachment.stagingKey(), + job.pipelineVersion(), Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt())); jdbc.update(""" INSERT INTO attachment_jobs @@ -129,7 +131,7 @@ public class PostgresAttachmentRepository implements AttachmentRepository { } @Override - public boolean claim(UUID jobId, UUID attachmentId, int attempt, + public boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, int attempt, UUID leaseToken, Duration leaseDuration) { int claimed = jdbc.update(""" UPDATE attachment_jobs @@ -138,22 +140,50 @@ public class PostgresAttachmentRepository implements AttachmentRepository { lease_until = clock_timestamp() + (? * interval '1 millisecond'), updated_at = clock_timestamp() WHERE id = ? AND attachment_id = ? + AND pipeline_version = ? AND (status = 'queued' OR (status = 'processing' AND lease_until < clock_timestamp())) - """, attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId); + AND EXISTS ( + SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ? + AND a.status IN ('queued', 'processing')) + """, attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId, + pipelineVersion, attachmentId, pipelineVersion); if (claimed == 0) { return false; } jdbc.update(""" UPDATE attachments SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() - WHERE id = ? AND status NOT IN ('ready', 'failed') - """, attachmentId); + WHERE id = ? AND pipeline_version = ? + AND status IN ('queued', 'processing') + """, attachmentId, pipelineVersion); return true; } @Override - public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken, + public boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken) { + return jdbc.update(""" + UPDATE attachment_jobs + SET before_effect_fault_injected = true, updated_at = clock_timestamp() + WHERE id = ? AND status = 'processing' AND lease_token = ? + AND before_effect_fault_injected = false + """, jobId, leaseToken) == 1; + } + + @Override + public boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken) { + return jdbc.update(""" + UPDATE attachment_jobs + SET after_effect_fault_injected = true, updated_at = clock_timestamp() + WHERE id = ? AND status = 'processing' AND lease_token = ? + AND after_effect_fault_injected = false + """, jobId, leaseToken) == 1; + } + + @Override + public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, + UUID leaseToken, int nextAttempt, String errorCode, Instant nextAttemptAt) { int updated = jdbc.update(""" UPDATE attachment_jobs @@ -161,17 +191,18 @@ public class PostgresAttachmentRepository implements AttachmentRepository { last_error_code = ?, next_attempt_at = ?, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE id = ? AND attachment_id = ? AND status = 'processing' - AND lease_token = ? + AND pipeline_version = ? AND lease_token = ? """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), - jobId, attachmentId, leaseToken); + jobId, attachmentId, pipelineVersion, leaseToken); if (updated == 0) { return false; } jdbc.update(""" UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() - WHERE id = ? AND status NOT IN ('ready', 'failed') - """, attachmentId); + WHERE id = ? AND pipeline_version = ? + AND status IN ('queued', 'processing') + """, attachmentId, pipelineVersion); return true; } @@ -192,8 +223,13 @@ public class PostgresAttachmentRepository implements AttachmentRepository { result_fault_injected = result_fault_injected OR ?, updated_at = clock_timestamp() WHERE id = ? AND attachment_id = ? AND status = 'processing' - AND lease_token = ? - """, triggerCrash, jobId, attachmentId, leaseToken); + AND pipeline_version = ? AND lease_token = ? + AND EXISTS ( + SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ? + AND a.status = 'processing') + """, triggerCrash, jobId, attachmentId, pipelineVersion, leaseToken, + attachmentId, pipelineVersion); if (completed == 0) { return new CompletionResult(false, false); } @@ -203,13 +239,7 @@ public class PostgresAttachmentRepository implements AttachmentRepository { (attachment_id, pipeline_version, name, content_type, size_bytes, width, height, sha256, object_key, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp()) - ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET - content_type = EXCLUDED.content_type, - size_bytes = EXCLUDED.size_bytes, - width = EXCLUDED.width, - height = EXCLUDED.height, - sha256 = EXCLUDED.sha256, - object_key = EXCLUDED.object_key + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING """, attachmentId, pipelineVersion, variant.name(), variant.contentType(), variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(), variant.objectKey()); @@ -217,29 +247,33 @@ public class PostgresAttachmentRepository implements AttachmentRepository { jdbc.update(""" UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() - WHERE id = ? AND status <> 'failed' - """, attachmentId); + WHERE id = ? AND pipeline_version = ? AND status = 'processing' + """, attachmentId, pipelineVersion); return new CompletionResult(true, triggerCrash); } @Override - public boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode) { + public boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion, + UUID leaseToken, String errorCode) { int updated = jdbc.update(""" UPDATE attachment_jobs SET status = 'failed', published = true, last_error_code = ?, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE id = ? AND attachment_id = ? AND status = 'processing' - AND lease_token = ? - """, errorCode, jobId, attachmentId, leaseToken); + AND pipeline_version = ? AND lease_token = ? + AND EXISTS (SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ?) + """, errorCode, jobId, attachmentId, pipelineVersion, leaseToken, + attachmentId, pipelineVersion); if (updated == 0) { return false; } jdbc.update(""" UPDATE attachments SET status = 'failed', error_code = ?, updated_at = clock_timestamp() - WHERE id = ? AND status <> 'ready' - """, errorCode, attachmentId); + WHERE id = ? AND pipeline_version = ? AND status <> 'ready' + """, errorCode, attachmentId, pipelineVersion); return true; } @@ -255,7 +289,8 @@ public class PostgresAttachmentRepository implements AttachmentRepository { } @Override - public boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt, + public boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, + int nextAttempt, String errorCode, Instant nextAttemptAt) { int updated = jdbc.update(""" UPDATE attachment_jobs @@ -263,33 +298,40 @@ public class PostgresAttachmentRepository implements AttachmentRepository { last_error_code = ?, next_attempt_at = ?, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE id = ? AND attachment_id = ? AND status = 'processing' - AND lease_until < clock_timestamp() - """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId); + AND pipeline_version = ? AND lease_until < clock_timestamp() + AND EXISTS (SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ?) + """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId, + pipelineVersion, attachmentId, pipelineVersion); if (updated == 1) { jdbc.update(""" UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() - WHERE id = ? AND status = 'processing' - """, attachmentId); + WHERE id = ? AND pipeline_version = ? AND status = 'processing' + """, attachmentId, pipelineVersion); } return updated == 1; } @Override - public boolean failExpired(UUID jobId, UUID attachmentId, String errorCode) { + public boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, + String errorCode) { int updated = jdbc.update(""" UPDATE attachment_jobs SET status = 'failed', published = true, last_error_code = ?, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE id = ? AND attachment_id = ? AND status = 'processing' - AND lease_until < clock_timestamp() - """, errorCode, jobId, attachmentId); + AND pipeline_version = ? AND lease_until < clock_timestamp() + AND EXISTS (SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ?) + """, errorCode, jobId, attachmentId, pipelineVersion, + attachmentId, pipelineVersion); if (updated == 1) { jdbc.update(""" UPDATE attachments SET status = 'failed', error_code = ?, updated_at = clock_timestamp() - WHERE id = ? AND status <> 'ready' - """, errorCode, attachmentId); + WHERE id = ? AND pipeline_version = ? AND status <> 'ready' + """, errorCode, attachmentId, pipelineVersion); } return updated == 1; } @@ -317,12 +359,14 @@ public class PostgresAttachmentRepository implements AttachmentRepository { return jdbc.query(""" SELECT name, content_type, size_bytes, width, height, sha256, object_key FROM attachment_variants - WHERE attachment_id = ? AND pipeline_version = 1 + WHERE attachment_id = ? + AND pipeline_version = ( + SELECT pipeline_version FROM attachments WHERE id = ?) ORDER BY name """, (rs, rowNum) -> new AttachmentVariant( rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"), rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"), - rs.getString("object_key")), attachmentId); + rs.getString("object_key")), attachmentId, attachmentId); } private static AttachmentJob mapJob(ResultSet rs) throws SQLException { diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java index 56d7bf63..cff26173 100644 --- a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java @@ -44,6 +44,7 @@ public class AttachmentService { private final Semaphore uploadPermits; private final long uploadPermitWaitMillis; private final boolean injectCrashAfterOutboxCommit; + private final int pipelineVersion; public AttachmentService( AttachmentRepository attachments, @@ -55,7 +56,8 @@ public class AttachmentService { @Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels, @Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads, @Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis, - @Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit + @Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit, + @Value("${messenger.jobs.pipeline-version:2}") int pipelineVersion ) { this.attachments = attachments; this.chats = chats; @@ -70,6 +72,11 @@ public class AttachmentService { this.uploadPermits = new Semaphore(maxConcurrentUploads, true); this.uploadPermitWaitMillis = uploadPermitWaitMillis; this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit; + if (pipelineVersion < AttachmentJob.MIN_PIPELINE_VERSION + || pipelineVersion > AttachmentJob.CURRENT_PIPELINE_VERSION) { + throw new IllegalArgumentException("Unsupported attachment pipeline version"); + } + this.pipelineVersion = pipelineVersion; } public Attachment upload(UUID chatId, @@ -119,7 +126,7 @@ public class AttachmentService { String objectKey = "attachments/" + id + "/original"; String stagingKey = "staging/" + id + "/original"; Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS); - AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, now); + AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, pipelineVersion, now); Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType, bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(), null, now, now); diff --git a/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java b/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java index 8e18b949..3b14c4b0 100644 --- a/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java +++ b/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java @@ -45,9 +45,13 @@ public class ReadinessService { } try { - // The schema-level probe also proves that the separate lab4 migration - // completed before this replica is allowed to serve business traffic. - jdbc.queryForObject("SELECT count(*) FROM attachments", Long.class); + // The schema-level probe proves that both the shared idempotency + // schema (lab3) and the attachment schema (lab4/lab5) completed + // before this replica is allowed to serve business traffic. + jdbc.queryForObject(""" + SELECT (SELECT count(*) FROM message_idempotency) + + (SELECT count(*) FROM attachments) + """, Long.class); markDependenciesAvailable(); return true; } catch (DataAccessException e) { diff --git a/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java b/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java index 571e475f..e7c8d48a 100644 --- a/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java +++ b/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java @@ -6,6 +6,7 @@ import com.example.mtusi_messenger.model.AttachmentVariant; import com.example.mtusi_messenger.queue.RetryPolicy; import com.example.mtusi_messenger.repository.AttachmentRepository; import com.example.mtusi_messenger.storage.ObjectStorage; +import com.example.mtusi_messenger.storage.ObjectStorageException; import com.fasterxml.jackson.databind.ObjectMapper; import com.rabbitmq.client.Channel; import jakarta.annotation.PreDestroy; @@ -47,6 +48,8 @@ public class AttachmentWorker { private final int maxAttempts; private final Duration jobTimeout; private final Duration leaseDuration; + private final boolean injectCrashBeforeSideEffect; + private final boolean injectCrashAfterSideEffect; private final boolean injectCrashAfterResultCommit; private final ExecutorService processingPool; private final Clock clock = Clock.systemUTC(); @@ -61,6 +64,8 @@ public class AttachmentWorker { @Value("${messenger.jobs.max-attempts:5}") int maxAttempts, @Value("${messenger.jobs.timeout:60s}") Duration jobTimeout, @Value("${messenger.jobs.lease-duration:90s}") Duration leaseDuration, + @Value("${messenger.jobs.fault-crash-before-side-effect:false}") boolean injectCrashBeforeSideEffect, + @Value("${messenger.jobs.fault-crash-after-side-effect:false}") boolean injectCrashAfterSideEffect, @Value("${messenger.jobs.fault-crash-after-result-commit:false}") boolean injectCrashAfterResultCommit, @Value("${spring.rabbitmq.listener.simple.concurrency:2}") int concurrency) { if (maxAttempts < 1 || jobTimeout.isZero() || jobTimeout.isNegative() @@ -76,6 +81,8 @@ public class AttachmentWorker { this.maxAttempts = maxAttempts; this.jobTimeout = jobTimeout; this.leaseDuration = leaseDuration; + this.injectCrashBeforeSideEffect = injectCrashBeforeSideEffect; + this.injectCrashAfterSideEffect = injectCrashAfterSideEffect; this.injectCrashAfterResultCommit = injectCrashAfterResultCommit; this.processingPool = Executors.newFixedThreadPool(concurrency, Thread.ofPlatform().name("attachment-image-", 0).factory()); @@ -103,7 +110,8 @@ public class AttachmentWorker { int attempt = Math.max(job.attempt(), attachments.currentAttempt(job.jobId())); UUID leaseToken = UUID.randomUUID(); boolean claimed = Boolean.TRUE.equals(transaction.execute(status -> - attachments.claim(job.jobId(), job.attachmentId(), attempt, leaseToken, leaseDuration))); + attachments.claim(job.jobId(), job.attachmentId(), job.pipelineVersion(), + attempt, leaseToken, leaseDuration))); if (!claimed) { // A live worker owns the lease, or a terminal state won the race. // If the owner died, the DB lease reaper will create a new outbox delivery. @@ -140,17 +148,23 @@ public class AttachmentWorker { } private ProcessOutcome process(AttachmentJob job, UUID leaseToken) { + if (injectCrashBeforeSideEffect + && attachments.markBeforeEffectFaultInjected(job.jobId(), leaseToken)) { + log.error("Fault injection: halting worker before object-storage side effect"); + Runtime.getRuntime().halt(84); + } Attachment attachment = attachments.findById(job.attachmentId()) .orElseThrow(() -> new ImageVariantProcessor.ProcessingException( "attachment_missing", true, "Attachment metadata is missing", null)); - byte[] original = storage.get(attachment.stagingKey()); + byte[] original = loadOriginal(attachment); if (original.length != attachment.sizeBytes() || !sha256(original).equals(attachment.sha256())) { throw new ImageVariantProcessor.ProcessingException( "invalid_image", true, "Staging checksum mismatch", null); } storage.put(attachment.objectKey(), original, attachment.contentType()); - List generated = images.createVariants(original); + List generated = + images.createVariants(original, job.pipelineVersion()); List variants = new ArrayList<>(generated.size()); for (ImageVariantProcessor.GeneratedVariant item : generated) { String objectKey = "attachments/" + attachment.id() + "/" @@ -161,6 +175,12 @@ public class AttachmentWorker { item.width(), item.height(), sha256(item.bytes()), objectKey)); } + if (injectCrashAfterSideEffect + && attachments.markAfterEffectFaultInjected(job.jobId(), leaseToken)) { + log.error("Fault injection: halting worker after object-storage side effect before result commit"); + Runtime.getRuntime().halt(85); + } + AttachmentRepository.CompletionResult completion = transaction.execute(status -> attachments.markReady(job.jobId(), job.attachmentId(), leaseToken, job.pipelineVersion(), variants, injectCrashAfterResultCommit)); @@ -175,11 +195,27 @@ public class AttachmentWorker { return new ProcessOutcome(completion.crashAfterCommit()); } + private byte[] loadOriginal(Attachment attachment) { + try { + return storage.get(attachment.stagingKey()); + } catch (ObjectStorageException missingStaging) { + try { + // A pipeline promotion reuses the immutable final original after + // the previous version safely removed its staging object. + return storage.get(attachment.objectKey()); + } catch (ObjectStorageException missingOriginal) { + missingOriginal.addSuppressed(missingStaging); + throw missingOriginal; + } + } + } + private void failPermanent( AttachmentJob job, UUID leaseToken, String errorCode, long deliveryTag, Channel channel) throws IOException { boolean failed = Boolean.TRUE.equals(transaction.execute(status -> - attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, errorCode))); + attachments.markFailed(job.jobId(), job.attachmentId(), job.pipelineVersion(), + leaseToken, errorCode))); if (failed) { channel.basicReject(deliveryTag, false); } else { @@ -193,8 +229,8 @@ public class AttachmentWorker { try { if (attempt >= maxAttempts) { boolean failed = Boolean.TRUE.equals(transaction.execute(status -> - attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, - "retry_exhausted"))); + attachments.markFailed(job.jobId(), job.attachmentId(), + job.pipelineVersion(), leaseToken, "retry_exhausted"))); if (failed) { log.error("Attachment job {} exhausted {} attempts: {}", job.jobId(), attempt, errorCode); @@ -208,7 +244,8 @@ public class AttachmentWorker { int nextAttempt = attempt + 1; boolean queued = Boolean.TRUE.equals(transaction.execute(status -> attachments.markQueuedForRetry( - job.jobId(), job.attachmentId(), leaseToken, nextAttempt, errorCode, + job.jobId(), job.attachmentId(), job.pipelineVersion(), + leaseToken, nextAttempt, errorCode, clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt))))); // The DB outbox is now the durable retry carrier; the current broker // delivery can be acknowledged without a DB/broker dual write. diff --git a/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java b/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java index cb077cae..f7e9faf9 100644 --- a/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java +++ b/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java @@ -45,7 +45,8 @@ public class ImageVariantProcessor { this.memoryWait = memoryWait; } - public List createVariants(byte[] original) { + public List createVariants(byte[] original, int pipelineVersion) { + PipelineProfile profile = PipelineProfile.forVersion(pipelineVersion); ImageInfo info = inspect(original); long estimatedBytes; try { @@ -70,7 +71,7 @@ public class ImageVariantProcessor { } try { - return transform(original); + return transform(original, profile); } finally { memoryPermits.release(permits); } @@ -106,7 +107,7 @@ public class ImageVariantProcessor { } } - private List transform(byte[] original) { + private List transform(byte[] original, PipelineProfile profile) { try { BufferedImage oriented = Thumbnails.of(new ByteArrayInputStream(original)) .useExifOrientation(true) @@ -118,17 +119,20 @@ public class ImageVariantProcessor { try { List variants = new ArrayList<>(3); BufferedImage thumbnail = fit(oriented, 256, BufferedImage.TYPE_INT_RGB, true); - variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", 0.85f)); + variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", + profile.thumbnailQuality())); thumbnail.flush(); BufferedImage crop = cropSquare(oriented, 128); - variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", 0.88f)); + variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", + profile.cropQuality())); crop.flush(); int webpType = oriented.getColorModel().hasAlpha() ? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB; BufferedImage optimized = fit(oriented, 1280, webpType, !oriented.getColorModel().hasAlpha()); - variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", 0.82f)); + variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", + profile.webpQuality())); optimized.flush(); return List.copyOf(variants); } finally { @@ -228,6 +232,18 @@ public class ImageVariantProcessor { private record ImageInfo(int width, int height, long pixels) { } + private record PipelineProfile( + int version, float thumbnailQuality, float cropQuality, float webpQuality) { + private static PipelineProfile forVersion(int version) { + return switch (version) { + case 1 -> new PipelineProfile(1, 0.85f, 0.88f, 0.82f); + case 2 -> new PipelineProfile(2, 0.87f, 0.90f, 0.84f); + default -> throw permanent( + "unsupported_pipeline", "Unsupported pipeline version " + version, null); + }; + } + } + public record GeneratedVariant(String name, String contentType, String extension, byte[] bytes, int width, int height) { } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 10f92def..b9e1ab64 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -95,6 +95,7 @@ messenger: reconcile-interval-millis: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:600000} reconcile-initial-delay-millis: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:60000} jobs: + pipeline-version: ${ATTACHMENT_PIPELINE_VERSION:2} exchange: ${ATTACHMENT_JOB_EXCHANGE:attachments.v1} queue: ${ATTACHMENT_JOB_QUEUE:attachments.process.level4.v1} routing-key: ${ATTACHMENT_JOB_ROUTING_KEY:attachment.process.v1} @@ -113,4 +114,6 @@ messenger: publish-confirm-timeout: ${JOB_PUBLISH_CONFIRM_TIMEOUT:5s} republish-interval: ${JOB_REPUBLISH_INTERVAL:5s} fault-crash-after-outbox-commit: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:false} + fault-crash-before-side-effect: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:false} + fault-crash-after-side-effect: ${FAULT_CRASH_AFTER_SIDE_EFFECT:false} fault-crash-after-result-commit: ${FAULT_CRASH_AFTER_RESULT_COMMIT:false} diff --git a/src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql b/src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql new file mode 100644 index 00000000..dd50c74f --- /dev/null +++ b/src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql @@ -0,0 +1,33 @@ +ALTER TABLE attachments + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD CONSTRAINT attachments_pipeline_version_chk CHECK (pipeline_version > 0); + +ALTER TABLE attachment_jobs + DROP CONSTRAINT attachment_jobs_attachment_id_key, + DROP CONSTRAINT attachment_jobs_status_chk, + ADD COLUMN before_effect_fault_injected boolean NOT NULL DEFAULT false, + ADD COLUMN after_effect_fault_injected boolean NOT NULL DEFAULT false, + ADD CONSTRAINT attachment_jobs_status_chk + CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')); + +CREATE UNIQUE INDEX attachment_jobs_attachment_pipeline_uk + ON attachment_jobs (attachment_id, pipeline_version); + +CREATE TABLE attachment_pipeline_promotions ( + id bigserial PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + from_version integer NOT NULL, + to_version integer NOT NULL, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE RESTRICT, + requested_by varchar(128) NOT NULL, + requested_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_pipeline_promotions_versions_chk + CHECK (from_version > 0 AND to_version > from_version), + CONSTRAINT attachment_pipeline_promotions_actor_chk + CHECK (char_length(requested_by) BETWEEN 1 AND 128), + CONSTRAINT attachment_pipeline_promotions_target_uk + UNIQUE (attachment_id, to_version) +); + +CREATE INDEX attachment_pipeline_promotions_attachment_idx + ON attachment_pipeline_promotions (attachment_id, requested_at); diff --git a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java index eb84fee2..5df23c69 100644 --- a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java +++ b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java @@ -55,7 +55,7 @@ class AttachmentServiceTest { new Chat(chatId, "test", List.of(ownerId), Instant.now())); when(repository.saveQueued(any(), any())).thenAnswer(invocation -> invocation.getArgument(0)); service = new AttachmentService(repository, chats, storage, publisher, transaction, - 10_485_760, 40_000_000, 3, 10_000, false); + 10_485_760, 40_000_000, 3, 10_000, false, 2); png = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png")); } @@ -73,7 +73,7 @@ class AttachmentServiceTest { assertFalse(attachment.objectKey().contains("photo.png")); assertArrayEquals(png, storage.objects.get(attachment.stagingKey())); assertFalse(storage.objects.containsKey(attachment.objectKey())); - verify(repository).saveQueued(eq(attachment), any()); + verify(repository).saveQueued(eq(attachment), argThat(job -> job.pipelineVersion() == 2)); verify(publisher).publish(any()); } diff --git a/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java b/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java index 11c56b5a..9601467b 100644 --- a/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java +++ b/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java @@ -9,10 +9,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.util.Map; +import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -30,7 +32,7 @@ class ImageVariantProcessorTest { 40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1)); Map variants = processor - .createVariants(original).stream() + .createVariants(original, 2).stream() .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, Function.identity())); @@ -49,12 +51,34 @@ class ImageVariantProcessorTest { 40_000_000, 1L * 1024 * 1024, Duration.ofSeconds(1)); var error = assertThrows(ImageVariantProcessor.ProcessingException.class, - () -> processor.createVariants(original)); + () -> processor.createVariants(original, 2)); assertTrue(error.permanent()); assertEquals("image_too_large", error.errorCode()); } + @Test + void versionsProduceSeparateDeterministicProfiles() { + ImageVariantProcessor processor = new ImageVariantProcessor( + 40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1)); + + Map version1 = processor + .createVariants(original, 1).stream() + .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, + Function.identity())); + Map version2 = processor + .createVariants(original, 2).stream() + .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, + Function.identity())); + + assertFalse(Arrays.equals(version1.get("optimized").bytes(), + version2.get("optimized").bytes())); + assertTrue(Arrays.equals(version2.get("optimized").bytes(), + processor.createVariants(original, 2).stream() + .filter(item -> item.name().equals("optimized")) + .findFirst().orElseThrow().bytes())); + } + private static void assertImage( ImageVariantProcessor.GeneratedVariant variant, String contentType, int width, int height) throws Exception { diff --git a/student_tests/README.md b/student_tests/README.md index 7b14c1ae..0bd7b296 100644 --- a/student_tests/README.md +++ b/student_tests/README.md @@ -15,10 +15,15 @@ The course-provided black-box suite remains in `tests/`; Spring tests remain in - `lab4_negative.py` — oversized/corrupt/MIME-spoofed uploads, unsafe filename, unknown attachment, foreign user and cross-chat binding; every user error is 4xx. - `concurrent_upload_memory.py` — 12 simultaneous ~6.5 MiB valid PNG uploads and - aggregate API container memory sampling through `docker stats`. + aggregate API container memory sampling through `docker stats` (lab5 `202` + upload contract). - `lab5_worker_recovery.py` — durable worker stop/restart, all three variants and idempotent duplicate delivery. - `lab5_fault_injection.py` — crashes after outbox commit and after result commit but before consumer acknowledgement. - `lab5_dlq_replay_lease.py` — permanent failure DLQ routing, guarded audited replay, and expired processing-lease recovery. +- `lab5_pipeline_idempotency.py` — versioned pipeline v1/v2, audited promotion, + `superseded` fencing, duplicate/reordered/stale deliveries, and crashes before + and after the S3 side effect. The script prints invariant checks and a final + `PASS` line; it uses `.env.example` for the disposable test stand by default. diff --git a/student_tests/concurrent_upload_memory.py b/student_tests/concurrent_upload_memory.py index a268da4d..ad17b230 100644 --- a/student_tests/concurrent_upload_memory.py +++ b/student_tests/concurrent_upload_memory.py @@ -104,7 +104,7 @@ def main(): barrier.wait(timeout=30) path = (f"/api/v1/chats/{chat['id']}/attachments" f"?filename=memory-{index}.png") - return client.call('POST', path, 201, actor=alice, raw=payload, + return client.call('POST', path, 202, actor=alice, raw=payload, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id'] monitor_thread = threading.Thread(target=monitor, daemon=True) diff --git a/student_tests/lab5_pipeline_idempotency.py b/student_tests/lab5_pipeline_idempotency.py new file mode 100644 index 00000000..75325248 --- /dev/null +++ b/student_tests/lab5_pipeline_idempotency.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +"""Prove version fencing and idempotent convergence under duplicate/crash faults.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] +FAULTS = ( + "FAULT_CRASH_AFTER_OUTBOX_COMMIT", + "FAULT_CRASH_BEFORE_SIDE_EFFECT", + "FAULT_CRASH_AFTER_SIDE_EFFECT", + "FAULT_CRASH_AFTER_RESULT_COMMIT", +) + + +def command(*args, capture=False, environment=None, check_result=True): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, text=True, capture_output=capture, + env=environment, check=check_result) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def env_value(name): + for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise Failure("missing " + name + " in .env.example") + + +def compose_environment(**overrides): + environment = os.environ.copy() + for name in FAULTS: + environment[name] = "false" + environment.update({name: str(value) for name, value in overrides.items()}) + return environment + + +def recreate(service, **overrides): + command( + "up", "-d", "--no-deps", "--force-recreate", service, + environment=compose_environment(**overrides)) + + +def wait_http(client, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready") + + +def wait_database(sql, expected, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if database_scalar(sql) == expected: + return + time.sleep(0.5) + raise Failure("database condition did not become " + expected + ": " + sql) + + +def job_row(attachment_id, version): + row = database_scalar( + "SELECT id || '|' || attempt || '|' || " + "to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM attachment_jobs WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(version) + ";") + check(bool(row), "pipeline job is missing") + return row.split("|") + + +def publish(attachment_id, version): + job_id, attempt, created_at = job_row(attachment_id, version) + payload = json.dumps({ + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": version, + "attempt": int(attempt), + "created_at": created_at, + }, separators=(",", ":")) + command( + "exec", "-T", "rabbitmq", "rabbitmqadmin", + "-u", env_value("RABBITMQ_USER"), "-p", env_value("RABBITMQ_PASSWORD"), + "publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"), + "routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"), + "payload=" + payload, + 'properties={"delivery_mode":2,"content_type":"application/json"}') + + +def promote(attachment_id, version): + subprocess.run( + ["powershell", "-NoProfile", "-File", str(ROOT / "scripts" / "promote-pipeline.ps1"), + "-AttachmentId", attachment_id, "-PipelineVersion", str(version), + "-RequestedBy", "idempotency-test", "-EnvFile", ".env.example"], + cwd=ROOT, check=True, text=True) + + +def variant_snapshot(attachment_id, version): + return database_scalar( + "SELECT string_agg(name || ':' || sha256 || ':' || object_key, ',' ORDER BY name) " + "FROM attachment_variants WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(version) + ";") + + +def job_status(attachment_id, version): + return database_scalar( + "SELECT status || '|' || published || '|' || coalesce(last_error_code, '') " + "FROM attachment_jobs WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(version) + ";") + + +def variant_count(attachment_id, version): + return database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + + attachment_id + "' AND pipeline_version=" + str(version) + ";") + + +def object_count(attachment_id): + script = ( + 'mc alias set local http://minio:9000 "$MINIO_ROOT_USER" ' + '"$MINIO_ROOT_PASSWORD" >/dev/null && ' + 'mc ls --recursive "local/$S3_BUCKET/attachments/' + attachment_id + + '/" 2>/dev/null | wc -l') + output = command( + "run", "--rm", "--no-deps", "--entrypoint", "/bin/sh", + "minio-init", "-c", script, capture=True) + return int(output.splitlines()[-1]) + + +def wait_marker(attachment_id, column, seconds=60): + wait_database( + "SELECT " + column + "::text FROM attachment_jobs WHERE attachment_id='" + + attachment_id + "' ORDER BY pipeline_version DESC LIMIT 1;", "true", seconds) + + +def wait_queue_empty(seconds=150): + deadline = time.monotonic() + seconds + marker = env_value("ATTACHMENT_JOB_QUEUE") + "\t0\t0" + while time.monotonic() < deadline: + listing = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", "messages_unacknowledged", capture=True) + if marker in listing: + return + time.sleep(0.5) + raise Failure("main queue did not drain") + + +def wait_worker_consumer(seconds=150): + deadline = time.monotonic() + seconds + queue = env_value("ATTACHMENT_JOB_QUEUE") + while time.monotonic() < deadline: + listing = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "consumers", capture=True) + for line in listing.splitlines(): + columns = line.split("\t") + if len(columns) == 2 and columns[0] == queue and int(columns[1]) > 0: + return + time.sleep(1) + raise Failure("worker did not register a RabbitMQ consumer") + + +def assert_ready_invariants(attachment_id, expected_version): + row = database_scalar( + "SELECT pipeline_version || '|' || status FROM attachments WHERE id='" + + attachment_id + "';") + check(row == str(expected_version) + "|ready", "active pipeline is not ready") + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(expected_version) + ";") == "3", + "active pipeline does not have exactly three variants") + check(database_scalar( + "SELECT count(*) FROM attachment_jobs WHERE attachment_id='" + attachment_id + + "' AND status IN ('queued','processing');") == "0", + "acknowledged attachment still has non-terminal work") + + +def fault_case(client, alice, bob, chat, flag, marker, expected_objects_before_commit): + recreate( + "worker", **{flag: "true", "JOB_TIMEOUT": "5s", "JOB_LEASE_DURATION": "8s"}) + wait_worker_consumer() + attachment, _ = client.attachment(alice, chat) + wait_marker(attachment["id"], marker, seconds=150) + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + + attachment["id"] + "';") == "0", + "result metadata committed before the injected pre-commit crash") + check(object_count(attachment["id"]) == expected_objects_before_commit, + "unexpected object count at injected fault point") + ready = client.wait_ready(bob, attachment, wait_seconds=150) + check(ready["status"] == "ready", "faulted job was lost") + assert_ready_invariants(attachment["id"], 2) + check(int(database_scalar( + "SELECT attempt FROM attachment_jobs WHERE attachment_id='" + + attachment["id"] + "' AND pipeline_version=2;")) >= 2, + "crashed job was not recovered through a later managed attempt") + check(object_count(attachment["id"]) == 4, + "idempotent retry did not converge to one deterministic object set") + recreate("worker") + return attachment["id"] + + +def main(): + client = Client("http://localhost:8080") + recreate("api", ATTACHMENT_PIPELINE_VERSION="1") + wait_http(client) + alice, bob, _, chat = client.bootstrap() + + # Complete v1, promote to v2, then deliver newer work before stale v1 and + # duplicate v2 messages. Arrival order intentionally differs from creation order. + versioned, _ = client.attachment(alice, chat) + client.wait_ready(bob, versioned, wait_seconds=120) + version1 = variant_snapshot(versioned["id"], 1) + check(bool(version1), "v1 result is missing") + command("stop", "worker") + promote(versioned["id"], 2) + publish(versioned["id"], 2) + publish(versioned["id"], 2) + publish(versioned["id"], 1) + command("up", "-d", "--no-deps", "worker") + client.wait_ready(bob, versioned, wait_seconds=120) + wait_queue_empty() + version2 = variant_snapshot(versioned["id"], 2) + check(bool(version2) and version2 != version1, + "v2 did not produce a distinct immutable versioned result") + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + + versioned["id"] + "';") == "6", "versioned rows were lost or duplicated") + stable = version2 + publish(versioned["id"], 1) + publish(versioned["id"], 2) + wait_queue_empty() + check(variant_snapshot(versioned["id"], 2) == stable, + "stale or duplicate delivery overwrote the active v2 result") + assert_ready_invariants(versioned["id"], 2) + + # Promote a still-queued v1 intent to v2. The old job must become + # superseded, and its already-durable message must be a harmless no-op. + command("stop", "worker") + superseded, _ = client.attachment(alice, chat) + check(job_status(superseded["id"], 1).startswith("queued|"), + "new upload did not start as a queued v1 job") + promote(superseded["id"], 2) + check(job_status(superseded["id"], 1) == "superseded|true|superseded_by_v2", + "queued v1 job was not fenced as superseded by v2") + check(variant_count(superseded["id"], 1) == "0", + "superseded job already has v1 result rows") + publish(superseded["id"], 1) + command("up", "-d", "--no-deps", "worker") + client.wait_ready(bob, superseded, wait_seconds=120) + wait_queue_empty() + check(variant_count(superseded["id"], 1) == "0", + "stale superseded v1 delivery created v1 result rows") + check(variant_count(superseded["id"], 2) == "3", + "promoted v2 job did not produce exactly three variants") + assert_ready_invariants(superseded["id"], 2) + + recreate("api", ATTACHMENT_PIPELINE_VERSION="2") + wait_http(client) + before_id = fault_case( + client, alice, bob, chat, "FAULT_CRASH_BEFORE_SIDE_EFFECT", + "before_effect_fault_injected", 0) + after_id = fault_case( + client, alice, bob, chat, "FAULT_CRASH_AFTER_SIDE_EFFECT", + "after_effect_fault_injected", 4) + + # Existing level-4 fault point proves redelivery after result COMMIT but before ack. + recreate("worker", FAULT_CRASH_AFTER_RESULT_COMMIT="true") + wait_worker_consumer() + committed, _ = client.attachment(alice, chat) + wait_marker(committed["id"], "result_fault_injected") + client.wait_ready(bob, committed, wait_seconds=120) + wait_queue_empty() + assert_ready_invariants(committed["id"], 2) + recreate("worker") + + confirmed = database_scalar( + "SELECT count(*) FROM attachments WHERE id IN ('" + versioned["id"] + "','" + + superseded["id"] + "','" + before_id + "','" + after_id + "','" + + committed["id"] + "') AND status='ready';") + check(confirmed == "5", "a confirmed task was lost") + print("PASS: version fencing, superseded, duplicate/reorder and " + "pre/post-side-effect crashes") + + +if __name__ == "__main__": + try: + run(main) + finally: + try: + recreate("api", ATTACHMENT_PIPELINE_VERSION="2") + recreate("worker") + except Exception: + pass