forked from mtusi2026/mtusi2026lab5
5
This commit is contained in:
@@ -14,6 +14,7 @@ ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1
|
||||
ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1
|
||||
ATTACHMENT_DLQ_QUEUE=attachments.failed.v1
|
||||
ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1
|
||||
ATTACHMENT_PIPELINE_VERSION=2
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
@@ -28,6 +29,8 @@ WORKER_MEMORY_WAIT=2s
|
||||
|
||||
# Fault injection is opt-in and intended only for an isolated test stand.
|
||||
FAULT_CRASH_AFTER_OUTBOX_COMMIT=false
|
||||
FAULT_CRASH_BEFORE_SIDE_EFFECT=false
|
||||
FAULT_CRASH_AFTER_SIDE_EFFECT=false
|
||||
FAULT_CRASH_AFTER_RESULT_COMMIT=false
|
||||
|
||||
S3_REGION=us-east-1
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка: 4
|
||||
- Автор, группа: Антипин Егор Иванович
|
||||
- Целевая оценка: **5**, трек 2 «Доказанная идемпотентность pipeline»
|
||||
- Commit/PR: TODO после публикации
|
||||
- Библиотеки: Spring AMQP, RabbitMQ Java client, Thumbnailator, webp-imageio, MinIO SDK, PostgreSQL/Flyway
|
||||
- Помощь ИИ: проектирование pipeline, реализация и подготовка тестов; результат проверен сборкой, unit- и black-box-тестами
|
||||
- Помощь ИИ: проектирование versioned pipeline, fencing и fault-injection, реализация, подготовка тестов и черновиков evidence; результат проверен сборкой, unit-, black-box- и аварийными тестами
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
@@ -18,24 +18,135 @@ docker compose --env-file .env up -d --build
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8080
|
||||
```
|
||||
|
||||
`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных выполняется только явной командой оператора с `--volumes`.
|
||||
Автоматические `student_tests/lab5_*.py` используют `.env.example` и рассчитаны
|
||||
на одноразовый учебный стенд; для них удобно поднимать Compose с тем же файлом:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env.example up -d --build
|
||||
python -X utf8 student_tests/lab5_pipeline_idempotency.py
|
||||
```
|
||||
|
||||
`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных
|
||||
выполняется только явной командой оператора с `--volumes`.
|
||||
|
||||
## Pipeline и гарантии
|
||||
|
||||
Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет неизменённый original в приватный `staging/<attachment_id>/original`, затем одной транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`. Запись job является transactional outbox. Фоновый publisher публикует persistent JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а HTTP `202` означает, что staging и намерение обработки уже устойчивы.
|
||||
Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет
|
||||
неизменённый original в приватный `staging/<attachment_id>/original`, затем одной
|
||||
транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`.
|
||||
Запись job является transactional outbox. Фоновый publisher публикует persistent
|
||||
JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job
|
||||
опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а
|
||||
HTTP `202` означает, что staging и намерение обработки уже устойчивы.
|
||||
|
||||
Отдельный `worker` захватывает job с lease, читает доверенный staging key из PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только `schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют. Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже `ready` job — no-op.
|
||||
Отдельный `worker` захватывает job с lease, читает доверенный staging key из
|
||||
PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только
|
||||
`schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`,
|
||||
`created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют.
|
||||
Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant
|
||||
rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже
|
||||
`ready` job — no-op.
|
||||
|
||||
Обязательные варианты pipeline version 1:
|
||||
Обязательные варианты для активной версии:
|
||||
|
||||
| Вариант | Правило | Формат/качество |
|
||||
| --- | --- | --- |
|
||||
| `original` | bytes и SHA-256 совпадают с upload | исходный MIME |
|
||||
| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality 0.85 |
|
||||
| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality 0.88 |
|
||||
| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality 0.82 |
|
||||
| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality зависит от версии |
|
||||
| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality зависит от версии |
|
||||
| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality зависит от версии |
|
||||
|
||||
Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных metadata.
|
||||
Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных
|
||||
metadata.
|
||||
|
||||
## Versioned pipeline v1/v2
|
||||
|
||||
Миграция `V7__versioned_pipeline_and_fault_points.sql` добавляет:
|
||||
|
||||
- `attachments.pipeline_version`;
|
||||
- уникальность `attachment_jobs(attachment_id, pipeline_version)`;
|
||||
- состояние `superseded`;
|
||||
- audit-таблицу `attachment_pipeline_promotions`;
|
||||
- маркеры fault-injection до и после side effect.
|
||||
|
||||
`ImageVariantProcessor` выбирает профиль по `pipeline_version`: v1 и v2 имеют
|
||||
разные quality для JPEG/WebP, поэтому результаты версий различаются побайтово.
|
||||
Ключи объектов версионированы: `thumbnail-v1.jpg`, `thumbnail-v2.jpg`,
|
||||
`optimized-v1.webp` и т.д.; общий `original` неизменяем. Старая версия физически
|
||||
не может перезаписать объект новой версии. Одна и та же `pipeline_version`
|
||||
обязана быть детерминированной: изменение quality/алгоритма оформляется новой
|
||||
версией, иначе `ON CONFLICT DO NOTHING` сохранит уже записанные variant rows.
|
||||
|
||||
Повышение версии — одна аудированная транзакция
|
||||
`scripts/promote-pipeline.ps1`/`.sh`:
|
||||
|
||||
1. блокирует attachment;
|
||||
2. переводит старые `queued/processing` jobs в `superseded` и `published=true`;
|
||||
3. создаёт новую job версии `N`;
|
||||
4. обновляет `attachments.pipeline_version` и `status=queued`;
|
||||
5. пишет `attachment_pipeline_promotions`.
|
||||
|
||||
Fencing выполняется в каждом переходе состояния: `claim`, `markReady`,
|
||||
`markFailed`, retry, expired-lease recovery и replay проверяют одновременно
|
||||
`job.pipeline_version`, `attachments.pipeline_version`, lease token и terminal
|
||||
state. Поэтому старая job после promotion не может ни получить lease, ни
|
||||
записать variants, ни перевести attachment в `ready`.
|
||||
|
||||
Если v1 уже удалил staging после безопасного COMMIT, worker версии v2 читает
|
||||
неизменяемый `attachments/<id>/original`; содержимое и SHA-256 те же.
|
||||
|
||||
## Fault injection и инварианты
|
||||
|
||||
| Точка | Поведение |
|
||||
| --- | --- |
|
||||
| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API падает после COMMIT attachment/outbox, до publish; republisher восстанавливает намерение |
|
||||
| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker падает до S3 PUT; lease истекает, reaper возвращает job в outbox |
|
||||
| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker падает после детерминированных S3 PUT, до result COMMIT; повтор сходится к тем же объектам |
|
||||
| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker падает после atomic `ready` COMMIT, до consumer ack; redelivery — no-op |
|
||||
|
||||
Все флаги по умолчанию выключены и предназначены только для изолированного
|
||||
стенда. Автоматический `student_tests/lab5_pipeline_idempotency.py` проверяет:
|
||||
|
||||
- duplicate: две одинаковые v2-доставки дают один результат;
|
||||
- reorder: v2 доставляется раньше stale v1;
|
||||
- stale: v1 после v2 не меняет snapshot активного результата;
|
||||
- superseded: повышение ещё queued v1 job отменяет её без variant rows;
|
||||
- crash до S3 side effect: 0 объектов, 0 result rows, восстановление через
|
||||
повторную попытку;
|
||||
- crash после S3 side effect: 4 объекта до COMMIT, после повтора — те же 4
|
||||
объекта и ровно 3 variant rows;
|
||||
- crash после result COMMIT: redelivery не дублирует variants;
|
||||
- все 5 подтверждённых вложений остаются `ready`.
|
||||
|
||||
Зафиксированные инварианты:
|
||||
|
||||
1. `ready` активной версии ⇒ ровно три variant rows активной версии и нет
|
||||
`queued/processing` jobs.
|
||||
2. Не более одной активной job на attachment.
|
||||
3. `(attachment_id, pipeline_version, name)` уникален; повтор не создаёт дубль.
|
||||
4. Versioned object keys не позволяют старой версии перезаписать новую.
|
||||
5. `pipeline_version` повышается только аудированной транзакцией; старая
|
||||
`queued/processing` job становится `superseded`.
|
||||
6. Подтверждённые задачи не теряются после duplicate/reorder/stale/crash.
|
||||
7. Нет variant rows версии больше активной.
|
||||
|
||||
## Почему это не exactly-once доставка
|
||||
|
||||
RabbitMQ обеспечивает **at-least-once**: сообщение может быть доставлено
|
||||
повторно после потери соединения или падения worker до ack; порядок сообщений не
|
||||
является бизнес-порядком. Publisher confirm подтверждает только приём брокером,
|
||||
а consumer ack идёт после PostgreSQL COMMIT. Между S3 PUT и COMMIT нет
|
||||
распределённой транзакции, поэтому crash в этом окне оставляет уже выполненные
|
||||
side effects и вызывает повторную обработку.
|
||||
|
||||
Идемпотентность достигнута на уровне **эффекта**, а не доставки: deterministic
|
||||
object keys, `ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady`
|
||||
приводят любые повторы к одному наблюдаемому состоянию активной версии. То же
|
||||
самое можно назвать **effectively-once** результатом при at-least-once
|
||||
доставке. Exactly-once доставка потребовала бы транзакционной связи RabbitMQ,
|
||||
PostgreSQL и S3 или эквивалентного распределённого протокола; в этой учебной
|
||||
архитектуре её нет, и тест сознательно демонстрирует лишние повторы S3 PUT,
|
||||
которые не меняют результат.
|
||||
|
||||
## Ограничения worker и retry
|
||||
|
||||
@@ -47,25 +158,49 @@ Upload разделён на durable-границы: API проверяет byte
|
||||
- Не более `JOB_MAX_ATTEMPTS=5`. Временные ошибки получают exponential backoff с jitter 50–100% и cap 30s. Ошибки decode/format/size постоянные и сразу становятся безопасным `failed/error_code`; exhausted retry — `retry_exhausted`.
|
||||
- Permanent malformed broker delivery и исчерпанная работа dead-lettered в durable `attachments.failed.v1`. Реестр failed остаётся в PostgreSQL.
|
||||
|
||||
Операторский replay выполняется `scripts/replay-failed.ps1 -JobId <uuid>` или `scripts/replay-failed.sh <uuid>`. Одна транзакция блокирует одновременно failed job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние. Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`.
|
||||
Операторский replay выполняется `scripts/replay-failed.ps1 -JobId <uuid>` или
|
||||
`scripts/replay-failed.sh <uuid>`. Одна транзакция блокирует одновременно failed
|
||||
job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние.
|
||||
Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`; replay
|
||||
также требует совпадения `pipeline_version`.
|
||||
|
||||
После безопасного COMMIT результата worker best-effort удаляет staging. `StagingCleaner` удаляет только неактивные объекты старше `STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён.
|
||||
После безопасного COMMIT результата worker best-effort удаляет staging.
|
||||
`StagingCleaner` удаляет только неактивные объекты старше
|
||||
`STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён. Отдельного
|
||||
GC для orphan-объектов старых версий после promotion «в полёте» нет: active
|
||||
результат они не перезаписывают, но место могут занимать.
|
||||
|
||||
## Fault injection и проверки
|
||||
## Проверки
|
||||
|
||||
| Критерий | Команда | Результат |
|
||||
| --- | --- | --- |
|
||||
| Unit tests | `gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' --tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'` | `BUILD SUCCESSFUL` |
|
||||
| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS |
|
||||
| Unit + integration | `gradlew test` | `BUILD SUCCESSFUL`, 30 tests |
|
||||
| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS, exit 0 |
|
||||
| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:8080` | PASS |
|
||||
| Worker stop, broker/API restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows |
|
||||
| COMMIT→publish и result COMMIT→ack crash | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи автоматически дошли до ready |
|
||||
| Трек 5.2 | `python -X utf8 student_tests/lab5_pipeline_idempotency.py` | PASS: version fencing, superseded, duplicate/reorder, pre/post-side-effect crashes |
|
||||
| Worker stop/restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows |
|
||||
| COMMIT→publish и result COMMIT→ack | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи дошли до ready |
|
||||
| DLQ, replay, lease | `python -X utf8 student_tests/lab5_dlq_replay_lease.py` | PASS; DLQ +1, одна audit-запись, ready не replayed, expired lease восстановлен |
|
||||
| Lab3: shared idempotency | `python -X utf8 student_tests/idempotency_concurrent.py` | PASS; 32 запроса, 2 реплики, одно сообщение |
|
||||
| Lab3: readiness gate | `python -X utf8 student_tests/readiness_gate.py` | PASS; обе реплики 503 на сломанной схеме и восстановились |
|
||||
| Lab3: failover каждой реплики | `python -X utf8 student_tests/failover_each.py ...` | PASS; 2 реплики, confirmed writes сохранены |
|
||||
| Lab3: rolling update | `python -X utf8 student_tests/rolling_update.py ...` | PASS; 4 instance ID, 335 успешных записей, 3 переходных отказа |
|
||||
| Lab4: negative | `python -X utf8 student_tests/lab4_negative.py` | PASS; ownership/MIME/corruption/limit/filename |
|
||||
| Lab4: upload memory | `python -X utf8 student_tests/concurrent_upload_memory.py ...` | PASS; 12/12 uploads, 2 API реплики |
|
||||
|
||||
Fault flags выключены по умолчанию. `FAULT_CRASH_AFTER_OUTBOX_COMMIT=true` один раз завершает API через `Runtime.halt(87)` после DB COMMIT до publish. После restart republisher находит `published=false`, и job достигает ready. `FAULT_CRASH_AFTER_RESULT_COMMIT=true` один раз завершает worker через `Runtime.halt(86)` после атомарного ready до consumer ack; RabbitMQ redelivery проверяет ready и завершает ту же job без второго набора вариантов.
|
||||
|
||||
Подробный журнал: `evidence/lab5-level4-2026-11-19.md`.
|
||||
Подробный журнал с SQL-инвариантами и JSON-агрегатами:
|
||||
`evidence/lab5-level5-2026-11-19.md`. Предыдущие уровни:
|
||||
`evidence/lab5-level4-2026-11-19.md`, `evidence/lab4-2026-11-11.md`.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Это уровень 4, не исследовательский трек 5: производительность для матрицы concurrency/prefetch не измерялась, multi-version reorder не исследовался. RabbitMQ, PostgreSQL и MinIO развёрнуты по одному экземпляру; volumes переживают container restart, но не потерю host/disk. At-least-once + идемпотентный эффект не является exactly-once доставкой.
|
||||
- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO;
|
||||
эксперимент не доказывает отказоустойчивость к потере host/диска.
|
||||
- Exactly-once доставки нет; доказан at-least-once + идемпотентный эффект.
|
||||
- Versioned orphan-объекты старых версий после promotion «в полёте» не
|
||||
перезаписывают active результат, но автоматического GC для них нет.
|
||||
- `promote-pipeline` — операторский скрипт с SQL-транзакцией, не публичный API.
|
||||
- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант;
|
||||
на чистом volume и для новых данных действует инвариант трёх вариантов.
|
||||
- Числа failover/rolling — проверка сохранности на одном Windows-хосте, не
|
||||
benchmark пропускной способности.
|
||||
|
||||
@@ -41,6 +41,7 @@ x-api-environment: &api-environment
|
||||
ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1}
|
||||
ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1}
|
||||
ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1}
|
||||
ATTACHMENT_PIPELINE_VERSION: ${ATTACHMENT_PIPELINE_VERSION:-2}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
@@ -258,6 +259,8 @@ services:
|
||||
MESSENGER_ROLE: worker
|
||||
SPRING_MAIN_WEB_APPLICATION_TYPE: none
|
||||
JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70
|
||||
FAULT_CRASH_BEFORE_SIDE_EFFECT: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:-false}
|
||||
FAULT_CRASH_AFTER_SIDE_EFFECT: ${FAULT_CRASH_AFTER_SIDE_EFFECT:-false}
|
||||
FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false}
|
||||
depends_on:
|
||||
migrate:
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"scenario": "sequential failover of each API replica",
|
||||
"replicas": 2,
|
||||
"results": [
|
||||
{
|
||||
"target_instance_id": "8e081627dfc3",
|
||||
"window_seconds": 15.775,
|
||||
"successful_reads": 3,
|
||||
"successful_writes": 4,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 0.444,
|
||||
"first_recovery_seconds": 5.018
|
||||
},
|
||||
{
|
||||
"target_instance_id": "95b5db854bee",
|
||||
"window_seconds": 16.774,
|
||||
"successful_reads": 3,
|
||||
"successful_writes": 3,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 0.358,
|
||||
"first_recovery_seconds": 6.15
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
# Lab5, трек 5.2 — доказанная идемпотентность pipeline
|
||||
|
||||
Дата: 2026-11-19. Исходная точка: `75d2f78` + рабочее дерево с изменениями lab5.
|
||||
Стенд: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23 (контейнеры и Gradle
|
||||
toolchain), Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, Redis 7.4,
|
||||
MinIO `bitnamilegacy/minio:2025.4.22-debian-12-r1`.
|
||||
|
||||
Все пользователи, сообщения и изображения синтетические. Секреты, session tokens,
|
||||
содержимое сообщений и signed URL в evidence не сохранялись.
|
||||
|
||||
## Гипотеза и инварианты
|
||||
|
||||
Гипотеза: при versioned pipeline и fencing по `(attachment_id, pipeline_version,
|
||||
lease)` повторная доставка, перестановка сообщений, crash в окнах до/после side
|
||||
effect и устаревшая job не теряют подтверждённую работу и не заменяют результат
|
||||
новой версии старым.
|
||||
|
||||
Проверяемые инварианты:
|
||||
|
||||
1. `attachments.status='ready'` для активной версии означает ровно три variant
|
||||
rows этой версии (`thumbnail`, `crop`, `optimized`) и отсутствие
|
||||
`queued/processing` jobs у вложения.
|
||||
2. Для вложения существует не более одной активной (`queued/processing`) job.
|
||||
3. Ключ variant rows — `(attachment_id, pipeline_version, name)`; повторная
|
||||
запись не создаёт дубль.
|
||||
4. Объектные ключи версионированы: `thumbnail-vN.jpg`, `crop-vN.jpg`,
|
||||
`optimized-vN.webp`; `original` общий и неизменяемый. Поэтому v1 не может
|
||||
перезаписать v2.
|
||||
5. `attachments.pipeline_version` монотонно повышается только аудированной
|
||||
транзакцией `promote-pipeline`; старая `queued/processing` job получает
|
||||
`superseded` и `published=true`.
|
||||
6. Все подтверждённые тестом задачи (`status='ready'`) остаются `ready` после
|
||||
duplicate/reorder/stale доставок.
|
||||
7. В active version нет variant rows с `pipeline_version` больше
|
||||
`attachments.pipeline_version`.
|
||||
8. Одна и та же `pipeline_version` детерминирована; изменение профиля
|
||||
оформляется новой версией, потому что `ON CONFLICT DO NOTHING` не перезаписывает
|
||||
уже зафиксированные variant rows.
|
||||
|
||||
## Команды и результаты
|
||||
|
||||
```text
|
||||
python -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
docker compose --env-file .env.example config --quiet
|
||||
exit 0
|
||||
|
||||
.\gradlew.bat test
|
||||
BUILD SUCCESSFUL (30 tests; full suite against live Compose PostgreSQL/Redis/MinIO)
|
||||
```
|
||||
|
||||
Для полного Gradle suite хранилища Compose не публикуют порты наружу; на время
|
||||
прогона использовались временные localhost-туннели к `postgres:5432`,
|
||||
`redis:6379`, `rabbitmq:5672`, `minio:9000`. После прогона туннели удалены.
|
||||
|
||||
### Основной эксперимент трека
|
||||
|
||||
```text
|
||||
python -X utf8 student_tests/lab5_pipeline_idempotency.py
|
||||
PASS: version fencing, superseded, duplicate/reorder and pre/post-side-effect crashes
|
||||
```
|
||||
|
||||
Тест автоматически:
|
||||
|
||||
1. создаёт v1 attachment, дожидается `ready` и снимает snapshot трёх variant rows;
|
||||
2. останавливает worker, повышает версию до v2 через `scripts/promote-pipeline`,
|
||||
публикует `v2, v2, v1` (порядок доставки не совпадает с порядком создания) и
|
||||
запускает worker;
|
||||
3. проверяет, что v2 дал отдельный результат, в БД ровно 6 rows (3 v1 + 3 v2),
|
||||
active version = 2, а повторные stale v1 / duplicate v2 не меняют snapshot;
|
||||
4. отдельно повышает ещё **queued** v1 job: она становится
|
||||
`superseded|published=true|superseded_by_v2`, v1 variant rows не появляются,
|
||||
v2 доходит до `ready` с тремя вариантами;
|
||||
5. вставляет crash до S3 side effect, после S3 side effect и после result
|
||||
COMMIT; каждый раз проверяет восстановление через lease/reaper/redelivery;
|
||||
6. проверяет, что все 5 подтверждённых вложений остались `ready`.
|
||||
|
||||
### Наблюдаемое состояние после прогона
|
||||
|
||||
Пример versioned-attachment (сокращённые ID приведены для читаемости):
|
||||
|
||||
```text
|
||||
attachment A (v1 -> v2, сначала завершён v1)
|
||||
active=2 ready
|
||||
job v1 ready published=t attempt=1
|
||||
job v2 ready published=t attempt=1
|
||||
variant keys: crop-v1, optimized-v1, thumbnail-v1,
|
||||
crop-v2, optimized-v2, thumbnail-v2
|
||||
|
||||
attachment B (queued v1 повышен до v2)
|
||||
active=2 ready
|
||||
job v1 superseded published=t last_error=superseded_by_v2
|
||||
job v2 ready published=t attempt=1
|
||||
variant keys: crop-v2, optimized-v2, thumbnail-v2 (v1 rows отсутствуют)
|
||||
```
|
||||
|
||||
SQL-инварианты по всем attachment, созданным 2026-11-19 в ходе финального
|
||||
прогона и регрессии lab5:
|
||||
|
||||
```text
|
||||
superseded_jobs|1
|
||||
today_promotions|9
|
||||
today_ready|55
|
||||
today_ready_bad_variants|0
|
||||
ready_with_nonterminal_job|0
|
||||
attachments_with_two_active_jobs|0
|
||||
duplicate_variant_keys|0
|
||||
stale_results_for_active_version|0
|
||||
```
|
||||
|
||||
Полный volume содержит также 3 attachment, созданных 2026-11-18 до миграции V7
|
||||
(тогда уровень 3 сохранял только original + thumbnail). Это исторические данные
|
||||
предыдущего стенда; они не являются результатом текущего versioned pipeline.
|
||||
Запросы инвариантов выше ограничены `created_at >= '2026-11-19'`; на чистом
|
||||
volume исторических строк нет.
|
||||
|
||||
### Точки отказа
|
||||
|
||||
| Fault | Где срабатывает | Наблюдение |
|
||||
| --- | --- | --- |
|
||||
| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker, до чтения staging/S3 PUT, `halt(84)` | 0 variant rows, 0 объектов; reaper возвращает job в `queued`, attempt увеличивается, задача доходит до `ready` |
|
||||
| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker, после 4 S3 PUT, до result COMMIT, `halt(85)` | objects=4, variant rows=0; повтор пишет те же deterministic keys и коммитит результат, objects остаётся 4 |
|
||||
| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker, после atomic `ready` COMMIT, до ack, `halt(86)` | redelivery видит `ready` и завершает no-op; 3 variant rows не дублируются |
|
||||
| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API, после COMMIT attachment/outbox, до publish, `halt(87)` | republisher находит `published=false` и публикует сохранённое намерение; задача доходит до `ready` |
|
||||
|
||||
Crash-точки включаются только переменными окружения и по умолчанию выключены;
|
||||
они предназначены для изолированного учебного стенда.
|
||||
|
||||
## Регрессия уровней 3–4
|
||||
|
||||
```text
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
python -X utf8 student_tests/lab5_worker_recovery.py
|
||||
PASS: stopped worker, durable restart recovery and duplicate delivery
|
||||
|
||||
python -X utf8 student_tests/lab5_fault_injection.py
|
||||
PASS: outbox commit/publish crash and result commit/ack crash recovered
|
||||
|
||||
python -X utf8 student_tests/lab5_dlq_replay_lease.py
|
||||
PASS: DLQ, guarded replay and expired lease recovery
|
||||
|
||||
python -X utf8 student_tests/idempotency_concurrent.py
|
||||
PASS: 32 concurrent requests used 2 replicas and created one message
|
||||
|
||||
python -X utf8 student_tests/readiness_gate.py
|
||||
PASS: 2 unhealthy API containers refused business traffic with 503 and recovered after schema restoration
|
||||
|
||||
python -X utf8 student_tests/failover_each.py --action-script scripts/stop-one.cmd \
|
||||
--restore-script scripts/restore-api.cmd --window 15 \
|
||||
--output evidence/failover-each-2026-11-19.json
|
||||
(JSON: 2 replicas sequentially stopped and restored; confirmed writes preserved)
|
||||
|
||||
python -X utf8 student_tests/rolling_update.py --action-script scripts/rolling-update.cmd \
|
||||
--duration 180 --concurrency 4 \
|
||||
--output evidence/rolling-update-2026-11-19.json
|
||||
(JSON: 4 observed replacement instance IDs; 335 successful writes, 3 transitional failures)
|
||||
|
||||
python -X utf8 student_tests/lab4_negative.py
|
||||
PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks
|
||||
|
||||
python -X utf8 student_tests/concurrent_upload_memory.py \
|
||||
--output evidence/upload-memory-2026-11-19.json
|
||||
PASS: concurrent upload memory experiment (12/12 uploads accepted, 2 API replicas)
|
||||
```
|
||||
|
||||
`readiness_gate.py` проверяет Google-подобный gate по обязательной схеме:
|
||||
`ReadinessService` теперь пробует обе таблицы — `message_idempotency` (lab3) и
|
||||
`attachments` (lab4/lab5). `concurrent_upload_memory.py` обновлён с lab4-контракта
|
||||
`201` на lab5-контракт `202`.
|
||||
|
||||
## Почему это at-least-once, а не exactly-once
|
||||
|
||||
RabbitMQ даёт at-least-once: сообщение может быть доставлено повторно после
|
||||
потери соединения или падения worker до ack; порядок между разными сообщениями
|
||||
не является бизнес-порядком. Поэтому duplicate и reorder — нормальные состояния,
|
||||
а не аномалии.
|
||||
|
||||
Наш pipeline делает **эффект** идемпотентным, но не делает доставку exactly-once:
|
||||
|
||||
- publisher confirm подтверждает только приём сообщения брокером, а consumer ack
|
||||
отправляется после PostgreSQL COMMIT; между S3 PUT и COMMIT нет распределённой
|
||||
транзакции;
|
||||
- crash после S3 PUT и до COMMIT оставляет объекты, а crash после COMMIT и до ack
|
||||
вызывает redelivery; оба окна специально инжектируются тестом;
|
||||
- повторная обработка может повторно выполнить S3 PUT, но deterministic keys,
|
||||
`ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady` не создают
|
||||
второго набора результатов и не меняют active version;
|
||||
- stale job ограничена `pipeline_version` и lease в каждом переходе состояния,
|
||||
поэтому не может записать результат поверх новой версии.
|
||||
|
||||
Корректная формулировка: **at-least-once delivery + idempotent effect =
|
||||
effectively-once наблюдаемый результат для активной версии**. Exactly-once
|
||||
доставка потребовала бы транзакционной связи RabbitMQ, PostgreSQL и S3 (или
|
||||
эквивалентного распределённого протокола), которой в этой учебной архитектуре
|
||||
нет и которая не нужна для данных инвариантов.
|
||||
|
||||
## Границы
|
||||
|
||||
- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO;
|
||||
эксперимент не доказывает отказоустойчивость к потере host/диска.
|
||||
- Versioned S3 objects старой версии после promotion «в полёте» могут остаться
|
||||
orphan-объектами: active result они не перезаписывают, но отдельного GC для
|
||||
versioned objects нет (чистится только staging).
|
||||
- `superseded`/promotion — операторская SQL-транзакция через скрипт, не публичный
|
||||
HTTP API.
|
||||
- Числа failover/rolling в JSON — проверка сохранности и восстановления на одном
|
||||
Windows-хосте, не benchmark пропускной способности; latency включает
|
||||
особенность localhost dual-stack в Docker Desktop.
|
||||
- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант;
|
||||
на чистом volume и для новых данных действует инвариант трёх вариантов.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"scenario": "sequential API replacement under closed-loop writes",
|
||||
"requested_duration_seconds": 180.0,
|
||||
"elapsed_seconds": 182.022,
|
||||
"rolling_action_seconds": 125.275,
|
||||
"concurrency": 4,
|
||||
"observed_instance_ids": [
|
||||
"431acab3a50c",
|
||||
"8e081627dfc3",
|
||||
"d67820ebd09d",
|
||||
"fdff9a67741c"
|
||||
],
|
||||
"successful_requests": 335,
|
||||
"failed_requests": 3,
|
||||
"error_ratio": 0.008876,
|
||||
"successful_rps": 1.84,
|
||||
"successful_latency_ms": {
|
||||
"p50": 2092.164,
|
||||
"p95": 2252.301,
|
||||
"p99": 3064.667
|
||||
},
|
||||
"note": "Closed-loop result on one host; success latency excludes failures."
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"concurrency": 12,
|
||||
"successful_uploads": 12,
|
||||
"payload_bytes_each": 6835833,
|
||||
"duration_seconds": 31.241,
|
||||
"api_replicas": 2,
|
||||
"memory_samples": 10,
|
||||
"baseline_api_memory_bytes": 665321471,
|
||||
"peak_api_memory_bytes": 833198489,
|
||||
"peak_increase_bytes": 167877018,
|
||||
"configured_max_concurrent_uploads_per_replica": 3,
|
||||
"configured_max_upload_bytes": 10485760,
|
||||
"configured_max_image_pixels": 40000000
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$AttachmentId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2)]
|
||||
[int]$PipelineVersion,
|
||||
[string]$RequestedBy = $env:USERNAME,
|
||||
[string]$EnvFile = ".env"
|
||||
)
|
||||
|
||||
$parsedAttachmentId = [guid]::Empty
|
||||
if (-not [guid]::TryParse($AttachmentId, [ref]$parsedAttachmentId)) {
|
||||
throw "AttachmentId must be a UUID"
|
||||
}
|
||||
if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') {
|
||||
throw "RequestedBy contains unsupported characters"
|
||||
}
|
||||
|
||||
$sql = @"
|
||||
WITH candidate AS (
|
||||
SELECT id, pipeline_version
|
||||
FROM attachments
|
||||
WHERE id = '$parsedAttachmentId'::uuid AND pipeline_version < $PipelineVersion
|
||||
FOR UPDATE
|
||||
), superseded AS (
|
||||
UPDATE attachment_jobs j
|
||||
SET status = 'superseded', published = true,
|
||||
last_error_code = 'superseded_by_v$PipelineVersion',
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c
|
||||
WHERE j.attachment_id = c.id AND j.pipeline_version < $PipelineVersion
|
||||
AND j.status IN ('queued', 'processing')
|
||||
RETURNING j.id
|
||||
), created AS (
|
||||
INSERT INTO attachment_jobs
|
||||
(id, attachment_id, pipeline_version, attempt, status, published,
|
||||
next_attempt_at, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), id, $PipelineVersion, 1, 'queued', false,
|
||||
clock_timestamp(), clock_timestamp(), clock_timestamp()
|
||||
FROM candidate
|
||||
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
|
||||
RETURNING id, attachment_id
|
||||
), promoted AS (
|
||||
UPDATE attachments a
|
||||
SET pipeline_version = $PipelineVersion, status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
FROM candidate c, created n
|
||||
WHERE a.id = c.id AND n.attachment_id = c.id
|
||||
RETURNING a.id
|
||||
), audited AS (
|
||||
INSERT INTO attachment_pipeline_promotions
|
||||
(attachment_id, from_version, to_version, job_id, requested_by)
|
||||
SELECT c.id, c.pipeline_version, $PipelineVersion, n.id, '$RequestedBy'
|
||||
FROM candidate c
|
||||
JOIN created n ON n.attachment_id = c.id
|
||||
JOIN promoted p ON p.id = c.id
|
||||
RETURNING job_id
|
||||
)
|
||||
SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');
|
||||
"@
|
||||
|
||||
$result = docker compose --env-file $EnvFile exec -T postgres `
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Pipeline promotion transaction failed"
|
||||
}
|
||||
|
||||
$last = ($result | Select-Object -Last 1).Trim()
|
||||
if ($last -eq "no-op") {
|
||||
Write-Output "No-op: attachment is missing, already at v$PipelineVersion, or that version already exists"
|
||||
} else {
|
||||
Write-Output "Promoted attachment $parsedAttachmentId to pipeline v$PipelineVersion with job $last"
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
attachment_id="${1:?usage: promote-pipeline.sh ATTACHMENT_UUID PIPELINE_VERSION [REQUESTED_BY] [ENV_FILE]}"
|
||||
pipeline_version="${2:?pipeline version is required}"
|
||||
requested_by="${3:-operator}"
|
||||
env_file="${4:-.env}"
|
||||
|
||||
case "$attachment_id" in
|
||||
????????-????-????-????-????????????) ;;
|
||||
*) echo "AttachmentId must be a UUID" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$pipeline_version" in
|
||||
1|2) ;;
|
||||
*) echo "PipelineVersion must be 1 or 2" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$requested_by" in
|
||||
*[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
sql="WITH candidate AS (
|
||||
SELECT id, pipeline_version FROM attachments
|
||||
WHERE id = '$attachment_id'::uuid AND pipeline_version < $pipeline_version FOR UPDATE
|
||||
), superseded AS (
|
||||
UPDATE attachment_jobs j SET status = 'superseded', published = true,
|
||||
last_error_code = 'superseded_by_v$pipeline_version',
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c WHERE j.attachment_id = c.id
|
||||
AND j.pipeline_version < $pipeline_version AND j.status IN ('queued', 'processing')
|
||||
RETURNING j.id
|
||||
), created AS (
|
||||
INSERT INTO attachment_jobs
|
||||
(id, attachment_id, pipeline_version, attempt, status, published,
|
||||
next_attempt_at, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), id, $pipeline_version, 1, 'queued', false,
|
||||
clock_timestamp(), clock_timestamp(), clock_timestamp() FROM candidate
|
||||
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
|
||||
RETURNING id, attachment_id
|
||||
), promoted AS (
|
||||
UPDATE attachments a SET pipeline_version = $pipeline_version,
|
||||
status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c, created n WHERE a.id = c.id AND n.attachment_id = c.id
|
||||
RETURNING a.id
|
||||
), audited AS (
|
||||
INSERT INTO attachment_pipeline_promotions
|
||||
(attachment_id, from_version, to_version, job_id, requested_by)
|
||||
SELECT c.id, c.pipeline_version, $pipeline_version, n.id, '$requested_by'
|
||||
FROM candidate c JOIN created n ON n.attachment_id = c.id
|
||||
JOIN promoted p ON p.id = c.id RETURNING job_id
|
||||
)
|
||||
SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');"
|
||||
|
||||
result="$(docker compose --env-file "$env_file" exec -T postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")"
|
||||
if [ "$result" = "no-op" ]; then
|
||||
echo "No-op: attachment is missing, already at v$pipeline_version, or that version already exists"
|
||||
else
|
||||
echo "Promoted attachment $attachment_id to pipeline v$pipeline_version with job $result"
|
||||
fi
|
||||
@@ -20,6 +20,7 @@ WITH candidate AS (
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = '$parsedJobId'::uuid
|
||||
AND j.status = 'failed' AND a.status = 'failed'
|
||||
AND j.pipeline_version = a.pipeline_version
|
||||
FOR UPDATE OF j, a
|
||||
), audited AS (
|
||||
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
|
||||
|
||||
@@ -17,6 +17,7 @@ sql="WITH candidate AS (
|
||||
SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed'
|
||||
AND j.pipeline_version = a.pipeline_version
|
||||
FOR UPDATE OF j, a
|
||||
), audited AS (
|
||||
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
|
||||
|
||||
@@ -15,19 +15,22 @@ public record AttachmentJob(
|
||||
@JsonProperty("created_at") Instant createdAt) {
|
||||
|
||||
public static final int SCHEMA_VERSION = 1;
|
||||
public static final int PIPELINE_VERSION = 1;
|
||||
public static final int MIN_PIPELINE_VERSION = 1;
|
||||
public static final int CURRENT_PIPELINE_VERSION = 2;
|
||||
public static final String TYPE = "attachment.process.v1";
|
||||
|
||||
public AttachmentJob {
|
||||
if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type)
|
||||
|| pipelineVersion < 1 || attempt < 1) {
|
||||
|| pipelineVersion < MIN_PIPELINE_VERSION
|
||||
|| pipelineVersion > CURRENT_PIPELINE_VERSION || attempt < 1) {
|
||||
throw new IllegalArgumentException("Unsupported attachment job message");
|
||||
}
|
||||
}
|
||||
|
||||
public static AttachmentJob first(UUID jobId, UUID attachmentId, Instant createdAt) {
|
||||
public static AttachmentJob first(
|
||||
UUID jobId, UUID attachmentId, int pipelineVersion, Instant createdAt) {
|
||||
return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId,
|
||||
PIPELINE_VERSION, 1, createdAt);
|
||||
pipelineVersion, 1, createdAt);
|
||||
}
|
||||
|
||||
public AttachmentJob nextAttempt(int nextAttempt) {
|
||||
|
||||
@@ -38,11 +38,13 @@ public class ExpiredLeaseReaper {
|
||||
try {
|
||||
transaction.executeWithoutResult(status -> {
|
||||
if (job.attempt() >= maxAttempts) {
|
||||
attachments.failExpired(job.jobId(), job.attachmentId(), "retry_exhausted");
|
||||
attachments.failExpired(job.jobId(), job.attachmentId(),
|
||||
job.pipelineVersion(), "retry_exhausted");
|
||||
} else {
|
||||
int nextAttempt = job.attempt() + 1;
|
||||
attachments.requeueExpired(
|
||||
job.jobId(), job.attachmentId(), nextAttempt, "processing_timeout",
|
||||
job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
nextAttempt, "processing_timeout",
|
||||
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -21,16 +21,20 @@ public interface AttachmentRepository {
|
||||
boolean markPublishFaultInjected(UUID jobId);
|
||||
List<AttachmentJob> findUnpublishedJobs(int limit);
|
||||
int currentAttempt(UUID jobId);
|
||||
boolean claim(UUID jobId, UUID attachmentId, int attempt, UUID leaseToken, Duration leaseDuration);
|
||||
boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken,
|
||||
boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
int attempt, UUID leaseToken, Duration leaseDuration);
|
||||
boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken);
|
||||
boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken);
|
||||
boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, UUID leaseToken,
|
||||
int nextAttempt, String errorCode, Instant nextAttemptAt);
|
||||
CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion,
|
||||
List<AttachmentVariant> variants, boolean injectCrashAfterCommit);
|
||||
boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode);
|
||||
boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
UUID leaseToken, String errorCode);
|
||||
List<AttachmentJob> findExpiredJobs(int limit);
|
||||
boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt,
|
||||
boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, int nextAttempt,
|
||||
String errorCode, Instant nextAttemptAt);
|
||||
boolean failExpired(UUID jobId, UUID attachmentId, String errorCode);
|
||||
boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, String errorCode);
|
||||
|
||||
record CompletionResult(boolean completed, boolean crashAfterCommit) {
|
||||
}
|
||||
|
||||
+83
-39
@@ -35,11 +35,13 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
jdbc.update("""
|
||||
INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, staging_key, status, error_code, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?)
|
||||
sha256, object_key, staging_key, pipeline_version,
|
||||
status, error_code, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?)
|
||||
""", attachment.id(), attachment.chatId(), attachment.ownerId(),
|
||||
attachment.filename(), attachment.contentType(), attachment.sizeBytes(),
|
||||
attachment.sha256(), attachment.objectKey(), attachment.stagingKey(),
|
||||
job.pipelineVersion(),
|
||||
Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt()));
|
||||
jdbc.update("""
|
||||
INSERT INTO attachment_jobs
|
||||
@@ -129,7 +131,7 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean claim(UUID jobId, UUID attachmentId, int attempt,
|
||||
public boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, int attempt,
|
||||
UUID leaseToken, Duration leaseDuration) {
|
||||
int claimed = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
@@ -138,22 +140,50 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
lease_until = clock_timestamp() + (? * interval '1 millisecond'),
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ?
|
||||
AND pipeline_version = ?
|
||||
AND (status = 'queued'
|
||||
OR (status = 'processing' AND lease_until < clock_timestamp()))
|
||||
""", attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId);
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?
|
||||
AND a.status IN ('queued', 'processing'))
|
||||
""", attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId,
|
||||
pipelineVersion, attachmentId, pipelineVersion);
|
||||
if (claimed == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status NOT IN ('ready', 'failed')
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ?
|
||||
AND status IN ('queued', 'processing')
|
||||
""", attachmentId, pipelineVersion);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, UUID leaseToken,
|
||||
public boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken) {
|
||||
return jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET before_effect_fault_injected = true, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing' AND lease_token = ?
|
||||
AND before_effect_fault_injected = false
|
||||
""", jobId, leaseToken) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken) {
|
||||
return jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET after_effect_fault_injected = true, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing' AND lease_token = ?
|
||||
AND after_effect_fault_injected = false
|
||||
""", jobId, leaseToken) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
UUID leaseToken,
|
||||
int nextAttempt, String errorCode, Instant nextAttemptAt) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
@@ -161,17 +191,18 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
last_error_code = ?, next_attempt_at = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
AND pipeline_version = ? AND lease_token = ?
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt),
|
||||
jobId, attachmentId, leaseToken);
|
||||
jobId, attachmentId, pipelineVersion, leaseToken);
|
||||
if (updated == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status NOT IN ('ready', 'failed')
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ?
|
||||
AND status IN ('queued', 'processing')
|
||||
""", attachmentId, pipelineVersion);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -192,8 +223,13 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
result_fault_injected = result_fault_injected OR ?,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", triggerCrash, jobId, attachmentId, leaseToken);
|
||||
AND pipeline_version = ? AND lease_token = ?
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?
|
||||
AND a.status = 'processing')
|
||||
""", triggerCrash, jobId, attachmentId, pipelineVersion, leaseToken,
|
||||
attachmentId, pipelineVersion);
|
||||
if (completed == 0) {
|
||||
return new CompletionResult(false, false);
|
||||
}
|
||||
@@ -203,13 +239,7 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
(attachment_id, pipeline_version, name, content_type, size_bytes,
|
||||
width, height, sha256, object_key, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp())
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET
|
||||
content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
object_key = EXCLUDED.object_key
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
|
||||
""", attachmentId, pipelineVersion, variant.name(), variant.contentType(),
|
||||
variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(),
|
||||
variant.objectKey());
|
||||
@@ -217,29 +247,33 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'ready', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'failed'
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status = 'processing'
|
||||
""", attachmentId, pipelineVersion);
|
||||
return new CompletionResult(true, triggerCrash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markFailed(UUID jobId, UUID attachmentId, UUID leaseToken, String errorCode) {
|
||||
public boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
UUID leaseToken, String errorCode) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', published = true, last_error_code = ?,
|
||||
lease_token = NULL, lease_until = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_token = ?
|
||||
""", errorCode, jobId, attachmentId, leaseToken);
|
||||
AND pipeline_version = ? AND lease_token = ?
|
||||
AND EXISTS (SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?)
|
||||
""", errorCode, jobId, attachmentId, pipelineVersion, leaseToken,
|
||||
attachmentId, pipelineVersion);
|
||||
if (updated == 0) {
|
||||
return false;
|
||||
}
|
||||
jdbc.update("""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = ?, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId, pipelineVersion);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -255,7 +289,8 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requeueExpired(UUID jobId, UUID attachmentId, int nextAttempt,
|
||||
public boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
int nextAttempt,
|
||||
String errorCode, Instant nextAttemptAt) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
@@ -263,33 +298,40 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
last_error_code = ?, next_attempt_at = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_until < clock_timestamp()
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId);
|
||||
AND pipeline_version = ? AND lease_until < clock_timestamp()
|
||||
AND EXISTS (SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?)
|
||||
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId,
|
||||
pipelineVersion, attachmentId, pipelineVersion);
|
||||
if (updated == 1) {
|
||||
jdbc.update("""
|
||||
UPDATE attachments SET status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status = 'processing'
|
||||
""", attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status = 'processing'
|
||||
""", attachmentId, pipelineVersion);
|
||||
}
|
||||
return updated == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean failExpired(UUID jobId, UUID attachmentId, String errorCode) {
|
||||
public boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion,
|
||||
String errorCode) {
|
||||
int updated = jdbc.update("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', published = true, last_error_code = ?,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = ? AND attachment_id = ? AND status = 'processing'
|
||||
AND lease_until < clock_timestamp()
|
||||
""", errorCode, jobId, attachmentId);
|
||||
AND pipeline_version = ? AND lease_until < clock_timestamp()
|
||||
AND EXISTS (SELECT 1 FROM attachments a
|
||||
WHERE a.id = ? AND a.pipeline_version = ?)
|
||||
""", errorCode, jobId, attachmentId, pipelineVersion,
|
||||
attachmentId, pipelineVersion);
|
||||
if (updated == 1) {
|
||||
jdbc.update("""
|
||||
UPDATE attachments SET status = 'failed', error_code = ?,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE id = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId);
|
||||
WHERE id = ? AND pipeline_version = ? AND status <> 'ready'
|
||||
""", errorCode, attachmentId, pipelineVersion);
|
||||
}
|
||||
return updated == 1;
|
||||
}
|
||||
@@ -317,12 +359,14 @@ public class PostgresAttachmentRepository implements AttachmentRepository {
|
||||
return jdbc.query("""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = ? AND pipeline_version = 1
|
||||
WHERE attachment_id = ?
|
||||
AND pipeline_version = (
|
||||
SELECT pipeline_version FROM attachments WHERE id = ?)
|
||||
ORDER BY name
|
||||
""", (rs, rowNum) -> new AttachmentVariant(
|
||||
rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"),
|
||||
rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"),
|
||||
rs.getString("object_key")), attachmentId);
|
||||
rs.getString("object_key")), attachmentId, attachmentId);
|
||||
}
|
||||
|
||||
private static AttachmentJob mapJob(ResultSet rs) throws SQLException {
|
||||
|
||||
@@ -44,6 +44,7 @@ public class AttachmentService {
|
||||
private final Semaphore uploadPermits;
|
||||
private final long uploadPermitWaitMillis;
|
||||
private final boolean injectCrashAfterOutboxCommit;
|
||||
private final int pipelineVersion;
|
||||
|
||||
public AttachmentService(
|
||||
AttachmentRepository attachments,
|
||||
@@ -55,7 +56,8 @@ public class AttachmentService {
|
||||
@Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels,
|
||||
@Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads,
|
||||
@Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis,
|
||||
@Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit
|
||||
@Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit,
|
||||
@Value("${messenger.jobs.pipeline-version:2}") int pipelineVersion
|
||||
) {
|
||||
this.attachments = attachments;
|
||||
this.chats = chats;
|
||||
@@ -70,6 +72,11 @@ public class AttachmentService {
|
||||
this.uploadPermits = new Semaphore(maxConcurrentUploads, true);
|
||||
this.uploadPermitWaitMillis = uploadPermitWaitMillis;
|
||||
this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit;
|
||||
if (pipelineVersion < AttachmentJob.MIN_PIPELINE_VERSION
|
||||
|| pipelineVersion > AttachmentJob.CURRENT_PIPELINE_VERSION) {
|
||||
throw new IllegalArgumentException("Unsupported attachment pipeline version");
|
||||
}
|
||||
this.pipelineVersion = pipelineVersion;
|
||||
}
|
||||
|
||||
public Attachment upload(UUID chatId,
|
||||
@@ -119,7 +126,7 @@ public class AttachmentService {
|
||||
String objectKey = "attachments/" + id + "/original";
|
||||
String stagingKey = "staging/" + id + "/original";
|
||||
Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS);
|
||||
AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, now);
|
||||
AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, pipelineVersion, now);
|
||||
Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType,
|
||||
bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(),
|
||||
null, now, now);
|
||||
|
||||
@@ -45,9 +45,13 @@ public class ReadinessService {
|
||||
}
|
||||
|
||||
try {
|
||||
// The schema-level probe also proves that the separate lab4 migration
|
||||
// completed before this replica is allowed to serve business traffic.
|
||||
jdbc.queryForObject("SELECT count(*) FROM attachments", Long.class);
|
||||
// The schema-level probe proves that both the shared idempotency
|
||||
// schema (lab3) and the attachment schema (lab4/lab5) completed
|
||||
// before this replica is allowed to serve business traffic.
|
||||
jdbc.queryForObject("""
|
||||
SELECT (SELECT count(*) FROM message_idempotency)
|
||||
+ (SELECT count(*) FROM attachments)
|
||||
""", Long.class);
|
||||
markDependenciesAvailable();
|
||||
return true;
|
||||
} catch (DataAccessException e) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.example.mtusi_messenger.model.AttachmentVariant;
|
||||
import com.example.mtusi_messenger.queue.RetryPolicy;
|
||||
import com.example.mtusi_messenger.repository.AttachmentRepository;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorage;
|
||||
import com.example.mtusi_messenger.storage.ObjectStorageException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
@@ -47,6 +48,8 @@ public class AttachmentWorker {
|
||||
private final int maxAttempts;
|
||||
private final Duration jobTimeout;
|
||||
private final Duration leaseDuration;
|
||||
private final boolean injectCrashBeforeSideEffect;
|
||||
private final boolean injectCrashAfterSideEffect;
|
||||
private final boolean injectCrashAfterResultCommit;
|
||||
private final ExecutorService processingPool;
|
||||
private final Clock clock = Clock.systemUTC();
|
||||
@@ -61,6 +64,8 @@ public class AttachmentWorker {
|
||||
@Value("${messenger.jobs.max-attempts:5}") int maxAttempts,
|
||||
@Value("${messenger.jobs.timeout:60s}") Duration jobTimeout,
|
||||
@Value("${messenger.jobs.lease-duration:90s}") Duration leaseDuration,
|
||||
@Value("${messenger.jobs.fault-crash-before-side-effect:false}") boolean injectCrashBeforeSideEffect,
|
||||
@Value("${messenger.jobs.fault-crash-after-side-effect:false}") boolean injectCrashAfterSideEffect,
|
||||
@Value("${messenger.jobs.fault-crash-after-result-commit:false}") boolean injectCrashAfterResultCommit,
|
||||
@Value("${spring.rabbitmq.listener.simple.concurrency:2}") int concurrency) {
|
||||
if (maxAttempts < 1 || jobTimeout.isZero() || jobTimeout.isNegative()
|
||||
@@ -76,6 +81,8 @@ public class AttachmentWorker {
|
||||
this.maxAttempts = maxAttempts;
|
||||
this.jobTimeout = jobTimeout;
|
||||
this.leaseDuration = leaseDuration;
|
||||
this.injectCrashBeforeSideEffect = injectCrashBeforeSideEffect;
|
||||
this.injectCrashAfterSideEffect = injectCrashAfterSideEffect;
|
||||
this.injectCrashAfterResultCommit = injectCrashAfterResultCommit;
|
||||
this.processingPool = Executors.newFixedThreadPool(concurrency,
|
||||
Thread.ofPlatform().name("attachment-image-", 0).factory());
|
||||
@@ -103,7 +110,8 @@ public class AttachmentWorker {
|
||||
int attempt = Math.max(job.attempt(), attachments.currentAttempt(job.jobId()));
|
||||
UUID leaseToken = UUID.randomUUID();
|
||||
boolean claimed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.claim(job.jobId(), job.attachmentId(), attempt, leaseToken, leaseDuration)));
|
||||
attachments.claim(job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
attempt, leaseToken, leaseDuration)));
|
||||
if (!claimed) {
|
||||
// A live worker owns the lease, or a terminal state won the race.
|
||||
// If the owner died, the DB lease reaper will create a new outbox delivery.
|
||||
@@ -140,17 +148,23 @@ public class AttachmentWorker {
|
||||
}
|
||||
|
||||
private ProcessOutcome process(AttachmentJob job, UUID leaseToken) {
|
||||
if (injectCrashBeforeSideEffect
|
||||
&& attachments.markBeforeEffectFaultInjected(job.jobId(), leaseToken)) {
|
||||
log.error("Fault injection: halting worker before object-storage side effect");
|
||||
Runtime.getRuntime().halt(84);
|
||||
}
|
||||
Attachment attachment = attachments.findById(job.attachmentId())
|
||||
.orElseThrow(() -> new ImageVariantProcessor.ProcessingException(
|
||||
"attachment_missing", true, "Attachment metadata is missing", null));
|
||||
byte[] original = storage.get(attachment.stagingKey());
|
||||
byte[] original = loadOriginal(attachment);
|
||||
if (original.length != attachment.sizeBytes() || !sha256(original).equals(attachment.sha256())) {
|
||||
throw new ImageVariantProcessor.ProcessingException(
|
||||
"invalid_image", true, "Staging checksum mismatch", null);
|
||||
}
|
||||
|
||||
storage.put(attachment.objectKey(), original, attachment.contentType());
|
||||
List<ImageVariantProcessor.GeneratedVariant> generated = images.createVariants(original);
|
||||
List<ImageVariantProcessor.GeneratedVariant> generated =
|
||||
images.createVariants(original, job.pipelineVersion());
|
||||
List<AttachmentVariant> variants = new ArrayList<>(generated.size());
|
||||
for (ImageVariantProcessor.GeneratedVariant item : generated) {
|
||||
String objectKey = "attachments/" + attachment.id() + "/"
|
||||
@@ -161,6 +175,12 @@ public class AttachmentWorker {
|
||||
item.width(), item.height(), sha256(item.bytes()), objectKey));
|
||||
}
|
||||
|
||||
if (injectCrashAfterSideEffect
|
||||
&& attachments.markAfterEffectFaultInjected(job.jobId(), leaseToken)) {
|
||||
log.error("Fault injection: halting worker after object-storage side effect before result commit");
|
||||
Runtime.getRuntime().halt(85);
|
||||
}
|
||||
|
||||
AttachmentRepository.CompletionResult completion = transaction.execute(status ->
|
||||
attachments.markReady(job.jobId(), job.attachmentId(), leaseToken,
|
||||
job.pipelineVersion(), variants, injectCrashAfterResultCommit));
|
||||
@@ -175,11 +195,27 @@ public class AttachmentWorker {
|
||||
return new ProcessOutcome(completion.crashAfterCommit());
|
||||
}
|
||||
|
||||
private byte[] loadOriginal(Attachment attachment) {
|
||||
try {
|
||||
return storage.get(attachment.stagingKey());
|
||||
} catch (ObjectStorageException missingStaging) {
|
||||
try {
|
||||
// A pipeline promotion reuses the immutable final original after
|
||||
// the previous version safely removed its staging object.
|
||||
return storage.get(attachment.objectKey());
|
||||
} catch (ObjectStorageException missingOriginal) {
|
||||
missingOriginal.addSuppressed(missingStaging);
|
||||
throw missingOriginal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void failPermanent(
|
||||
AttachmentJob job, UUID leaseToken, String errorCode,
|
||||
long deliveryTag, Channel channel) throws IOException {
|
||||
boolean failed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken, errorCode)));
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
leaseToken, errorCode)));
|
||||
if (failed) {
|
||||
channel.basicReject(deliveryTag, false);
|
||||
} else {
|
||||
@@ -193,8 +229,8 @@ public class AttachmentWorker {
|
||||
try {
|
||||
if (attempt >= maxAttempts) {
|
||||
boolean failed = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(), leaseToken,
|
||||
"retry_exhausted")));
|
||||
attachments.markFailed(job.jobId(), job.attachmentId(),
|
||||
job.pipelineVersion(), leaseToken, "retry_exhausted")));
|
||||
if (failed) {
|
||||
log.error("Attachment job {} exhausted {} attempts: {}",
|
||||
job.jobId(), attempt, errorCode);
|
||||
@@ -208,7 +244,8 @@ public class AttachmentWorker {
|
||||
int nextAttempt = attempt + 1;
|
||||
boolean queued = Boolean.TRUE.equals(transaction.execute(status ->
|
||||
attachments.markQueuedForRetry(
|
||||
job.jobId(), job.attachmentId(), leaseToken, nextAttempt, errorCode,
|
||||
job.jobId(), job.attachmentId(), job.pipelineVersion(),
|
||||
leaseToken, nextAttempt, errorCode,
|
||||
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)))));
|
||||
// The DB outbox is now the durable retry carrier; the current broker
|
||||
// delivery can be acknowledged without a DB/broker dual write.
|
||||
|
||||
@@ -45,7 +45,8 @@ public class ImageVariantProcessor {
|
||||
this.memoryWait = memoryWait;
|
||||
}
|
||||
|
||||
public List<GeneratedVariant> createVariants(byte[] original) {
|
||||
public List<GeneratedVariant> createVariants(byte[] original, int pipelineVersion) {
|
||||
PipelineProfile profile = PipelineProfile.forVersion(pipelineVersion);
|
||||
ImageInfo info = inspect(original);
|
||||
long estimatedBytes;
|
||||
try {
|
||||
@@ -70,7 +71,7 @@ public class ImageVariantProcessor {
|
||||
}
|
||||
|
||||
try {
|
||||
return transform(original);
|
||||
return transform(original, profile);
|
||||
} finally {
|
||||
memoryPermits.release(permits);
|
||||
}
|
||||
@@ -106,7 +107,7 @@ public class ImageVariantProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
private List<GeneratedVariant> transform(byte[] original) {
|
||||
private List<GeneratedVariant> transform(byte[] original, PipelineProfile profile) {
|
||||
try {
|
||||
BufferedImage oriented = Thumbnails.of(new ByteArrayInputStream(original))
|
||||
.useExifOrientation(true)
|
||||
@@ -118,17 +119,20 @@ public class ImageVariantProcessor {
|
||||
try {
|
||||
List<GeneratedVariant> variants = new ArrayList<>(3);
|
||||
BufferedImage thumbnail = fit(oriented, 256, BufferedImage.TYPE_INT_RGB, true);
|
||||
variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", 0.85f));
|
||||
variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg",
|
||||
profile.thumbnailQuality()));
|
||||
thumbnail.flush();
|
||||
|
||||
BufferedImage crop = cropSquare(oriented, 128);
|
||||
variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", 0.88f));
|
||||
variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg",
|
||||
profile.cropQuality()));
|
||||
crop.flush();
|
||||
|
||||
int webpType = oriented.getColorModel().hasAlpha()
|
||||
? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB;
|
||||
BufferedImage optimized = fit(oriented, 1280, webpType, !oriented.getColorModel().hasAlpha());
|
||||
variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", 0.82f));
|
||||
variants.add(encode("optimized", "image/webp", "webp", optimized, "webp",
|
||||
profile.webpQuality()));
|
||||
optimized.flush();
|
||||
return List.copyOf(variants);
|
||||
} finally {
|
||||
@@ -228,6 +232,18 @@ public class ImageVariantProcessor {
|
||||
private record ImageInfo(int width, int height, long pixels) {
|
||||
}
|
||||
|
||||
private record PipelineProfile(
|
||||
int version, float thumbnailQuality, float cropQuality, float webpQuality) {
|
||||
private static PipelineProfile forVersion(int version) {
|
||||
return switch (version) {
|
||||
case 1 -> new PipelineProfile(1, 0.85f, 0.88f, 0.82f);
|
||||
case 2 -> new PipelineProfile(2, 0.87f, 0.90f, 0.84f);
|
||||
default -> throw permanent(
|
||||
"unsupported_pipeline", "Unsupported pipeline version " + version, null);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public record GeneratedVariant(String name, String contentType, String extension,
|
||||
byte[] bytes, int width, int height) {
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ messenger:
|
||||
reconcile-interval-millis: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:600000}
|
||||
reconcile-initial-delay-millis: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:60000}
|
||||
jobs:
|
||||
pipeline-version: ${ATTACHMENT_PIPELINE_VERSION:2}
|
||||
exchange: ${ATTACHMENT_JOB_EXCHANGE:attachments.v1}
|
||||
queue: ${ATTACHMENT_JOB_QUEUE:attachments.process.level4.v1}
|
||||
routing-key: ${ATTACHMENT_JOB_ROUTING_KEY:attachment.process.v1}
|
||||
@@ -113,4 +114,6 @@ messenger:
|
||||
publish-confirm-timeout: ${JOB_PUBLISH_CONFIRM_TIMEOUT:5s}
|
||||
republish-interval: ${JOB_REPUBLISH_INTERVAL:5s}
|
||||
fault-crash-after-outbox-commit: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:false}
|
||||
fault-crash-before-side-effect: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:false}
|
||||
fault-crash-after-side-effect: ${FAULT_CRASH_AFTER_SIDE_EFFECT:false}
|
||||
fault-crash-after-result-commit: ${FAULT_CRASH_AFTER_RESULT_COMMIT:false}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
|
||||
ADD CONSTRAINT attachments_pipeline_version_chk CHECK (pipeline_version > 0);
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
DROP CONSTRAINT attachment_jobs_attachment_id_key,
|
||||
DROP CONSTRAINT attachment_jobs_status_chk,
|
||||
ADD COLUMN before_effect_fault_injected boolean NOT NULL DEFAULT false,
|
||||
ADD COLUMN after_effect_fault_injected boolean NOT NULL DEFAULT false,
|
||||
ADD CONSTRAINT attachment_jobs_status_chk
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded'));
|
||||
|
||||
CREATE UNIQUE INDEX attachment_jobs_attachment_pipeline_uk
|
||||
ON attachment_jobs (attachment_id, pipeline_version);
|
||||
|
||||
CREATE TABLE attachment_pipeline_promotions (
|
||||
id bigserial PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
from_version integer NOT NULL,
|
||||
to_version integer NOT NULL,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE RESTRICT,
|
||||
requested_by varchar(128) NOT NULL,
|
||||
requested_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_pipeline_promotions_versions_chk
|
||||
CHECK (from_version > 0 AND to_version > from_version),
|
||||
CONSTRAINT attachment_pipeline_promotions_actor_chk
|
||||
CHECK (char_length(requested_by) BETWEEN 1 AND 128),
|
||||
CONSTRAINT attachment_pipeline_promotions_target_uk
|
||||
UNIQUE (attachment_id, to_version)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_pipeline_promotions_attachment_idx
|
||||
ON attachment_pipeline_promotions (attachment_id, requested_at);
|
||||
@@ -55,7 +55,7 @@ class AttachmentServiceTest {
|
||||
new Chat(chatId, "test", List.of(ownerId), Instant.now()));
|
||||
when(repository.saveQueued(any(), any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
service = new AttachmentService(repository, chats, storage, publisher, transaction,
|
||||
10_485_760, 40_000_000, 3, 10_000, false);
|
||||
10_485_760, 40_000_000, 3, 10_000, false, 2);
|
||||
png = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png"));
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ class AttachmentServiceTest {
|
||||
assertFalse(attachment.objectKey().contains("photo.png"));
|
||||
assertArrayEquals(png, storage.objects.get(attachment.stagingKey()));
|
||||
assertFalse(storage.objects.containsKey(attachment.objectKey()));
|
||||
verify(repository).saveQueued(eq(attachment), any());
|
||||
verify(repository).saveQueued(eq(attachment), argThat(job -> job.pipelineVersion() == 2));
|
||||
verify(publisher).publish(any());
|
||||
}
|
||||
|
||||
|
||||
@@ -9,10 +9,12 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.Arrays;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
@@ -30,7 +32,7 @@ class ImageVariantProcessorTest {
|
||||
40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1));
|
||||
|
||||
Map<String, ImageVariantProcessor.GeneratedVariant> variants = processor
|
||||
.createVariants(original).stream()
|
||||
.createVariants(original, 2).stream()
|
||||
.collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name,
|
||||
Function.identity()));
|
||||
|
||||
@@ -49,12 +51,34 @@ class ImageVariantProcessorTest {
|
||||
40_000_000, 1L * 1024 * 1024, Duration.ofSeconds(1));
|
||||
|
||||
var error = assertThrows(ImageVariantProcessor.ProcessingException.class,
|
||||
() -> processor.createVariants(original));
|
||||
() -> processor.createVariants(original, 2));
|
||||
|
||||
assertTrue(error.permanent());
|
||||
assertEquals("image_too_large", error.errorCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void versionsProduceSeparateDeterministicProfiles() {
|
||||
ImageVariantProcessor processor = new ImageVariantProcessor(
|
||||
40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1));
|
||||
|
||||
Map<String, ImageVariantProcessor.GeneratedVariant> version1 = processor
|
||||
.createVariants(original, 1).stream()
|
||||
.collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name,
|
||||
Function.identity()));
|
||||
Map<String, ImageVariantProcessor.GeneratedVariant> version2 = processor
|
||||
.createVariants(original, 2).stream()
|
||||
.collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name,
|
||||
Function.identity()));
|
||||
|
||||
assertFalse(Arrays.equals(version1.get("optimized").bytes(),
|
||||
version2.get("optimized").bytes()));
|
||||
assertTrue(Arrays.equals(version2.get("optimized").bytes(),
|
||||
processor.createVariants(original, 2).stream()
|
||||
.filter(item -> item.name().equals("optimized"))
|
||||
.findFirst().orElseThrow().bytes()));
|
||||
}
|
||||
|
||||
private static void assertImage(
|
||||
ImageVariantProcessor.GeneratedVariant variant,
|
||||
String contentType, int width, int height) throws Exception {
|
||||
|
||||
@@ -15,10 +15,15 @@ The course-provided black-box suite remains in `tests/`; Spring tests remain in
|
||||
- `lab4_negative.py` — oversized/corrupt/MIME-spoofed uploads, unsafe filename,
|
||||
unknown attachment, foreign user and cross-chat binding; every user error is 4xx.
|
||||
- `concurrent_upload_memory.py` — 12 simultaneous ~6.5 MiB valid PNG uploads and
|
||||
aggregate API container memory sampling through `docker stats`.
|
||||
aggregate API container memory sampling through `docker stats` (lab5 `202`
|
||||
upload contract).
|
||||
- `lab5_worker_recovery.py` — durable worker stop/restart, all three variants and
|
||||
idempotent duplicate delivery.
|
||||
- `lab5_fault_injection.py` — crashes after outbox commit and after result commit
|
||||
but before consumer acknowledgement.
|
||||
- `lab5_dlq_replay_lease.py` — permanent failure DLQ routing, guarded audited
|
||||
replay, and expired processing-lease recovery.
|
||||
- `lab5_pipeline_idempotency.py` — versioned pipeline v1/v2, audited promotion,
|
||||
`superseded` fencing, duplicate/reordered/stale deliveries, and crashes before
|
||||
and after the S3 side effect. The script prints invariant checks and a final
|
||||
`PASS` line; it uses `.env.example` for the disposable test stand by default.
|
||||
|
||||
@@ -104,7 +104,7 @@ def main():
|
||||
barrier.wait(timeout=30)
|
||||
path = (f"/api/v1/chats/{chat['id']}/attachments"
|
||||
f"?filename=memory-{index}.png")
|
||||
return client.call('POST', path, 201, actor=alice, raw=payload,
|
||||
return client.call('POST', path, 202, actor=alice, raw=payload,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id']
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor, daemon=True)
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove version fencing and idempotent convergence under duplicate/crash faults."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env.example"]
|
||||
FAULTS = (
|
||||
"FAULT_CRASH_AFTER_OUTBOX_COMMIT",
|
||||
"FAULT_CRASH_BEFORE_SIDE_EFFECT",
|
||||
"FAULT_CRASH_AFTER_SIDE_EFFECT",
|
||||
"FAULT_CRASH_AFTER_RESULT_COMMIT",
|
||||
)
|
||||
|
||||
|
||||
def command(*args, capture=False, environment=None, check_result=True):
|
||||
result = subprocess.run(
|
||||
[*COMPOSE, *args], cwd=ROOT, text=True, capture_output=capture,
|
||||
env=environment, check=check_result)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def database_scalar(sql):
|
||||
return command(
|
||||
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", sql, capture=True)
|
||||
|
||||
|
||||
def env_value(name):
|
||||
for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith(name + "="):
|
||||
return line.split("=", 1)[1]
|
||||
raise Failure("missing " + name + " in .env.example")
|
||||
|
||||
|
||||
def compose_environment(**overrides):
|
||||
environment = os.environ.copy()
|
||||
for name in FAULTS:
|
||||
environment[name] = "false"
|
||||
environment.update({name: str(value) for name, value in overrides.items()})
|
||||
return environment
|
||||
|
||||
|
||||
def recreate(service, **overrides):
|
||||
command(
|
||||
"up", "-d", "--no-deps", "--force-recreate", service,
|
||||
environment=compose_environment(**overrides))
|
||||
|
||||
|
||||
def wait_http(client, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API did not become ready")
|
||||
|
||||
|
||||
def wait_database(sql, expected, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if database_scalar(sql) == expected:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise Failure("database condition did not become " + expected + ": " + sql)
|
||||
|
||||
|
||||
def job_row(attachment_id, version):
|
||||
row = database_scalar(
|
||||
"SELECT id || '|' || attempt || '|' || "
|
||||
"to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') "
|
||||
"FROM attachment_jobs WHERE attachment_id='" + attachment_id
|
||||
+ "' AND pipeline_version=" + str(version) + ";")
|
||||
check(bool(row), "pipeline job is missing")
|
||||
return row.split("|")
|
||||
|
||||
|
||||
def publish(attachment_id, version):
|
||||
job_id, attempt, created_at = job_row(attachment_id, version)
|
||||
payload = json.dumps({
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": attachment_id,
|
||||
"pipeline_version": version,
|
||||
"attempt": int(attempt),
|
||||
"created_at": created_at,
|
||||
}, separators=(",", ":"))
|
||||
command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqadmin",
|
||||
"-u", env_value("RABBITMQ_USER"), "-p", env_value("RABBITMQ_PASSWORD"),
|
||||
"publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"),
|
||||
"routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"),
|
||||
"payload=" + payload,
|
||||
'properties={"delivery_mode":2,"content_type":"application/json"}')
|
||||
|
||||
|
||||
def promote(attachment_id, version):
|
||||
subprocess.run(
|
||||
["powershell", "-NoProfile", "-File", str(ROOT / "scripts" / "promote-pipeline.ps1"),
|
||||
"-AttachmentId", attachment_id, "-PipelineVersion", str(version),
|
||||
"-RequestedBy", "idempotency-test", "-EnvFile", ".env.example"],
|
||||
cwd=ROOT, check=True, text=True)
|
||||
|
||||
|
||||
def variant_snapshot(attachment_id, version):
|
||||
return database_scalar(
|
||||
"SELECT string_agg(name || ':' || sha256 || ':' || object_key, ',' ORDER BY name) "
|
||||
"FROM attachment_variants WHERE attachment_id='" + attachment_id
|
||||
+ "' AND pipeline_version=" + str(version) + ";")
|
||||
|
||||
|
||||
def job_status(attachment_id, version):
|
||||
return database_scalar(
|
||||
"SELECT status || '|' || published || '|' || coalesce(last_error_code, '') "
|
||||
"FROM attachment_jobs WHERE attachment_id='" + attachment_id
|
||||
+ "' AND pipeline_version=" + str(version) + ";")
|
||||
|
||||
|
||||
def variant_count(attachment_id, version):
|
||||
return database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='"
|
||||
+ attachment_id + "' AND pipeline_version=" + str(version) + ";")
|
||||
|
||||
|
||||
def object_count(attachment_id):
|
||||
script = (
|
||||
'mc alias set local http://minio:9000 "$MINIO_ROOT_USER" '
|
||||
'"$MINIO_ROOT_PASSWORD" >/dev/null && '
|
||||
'mc ls --recursive "local/$S3_BUCKET/attachments/' + attachment_id
|
||||
+ '/" 2>/dev/null | wc -l')
|
||||
output = command(
|
||||
"run", "--rm", "--no-deps", "--entrypoint", "/bin/sh",
|
||||
"minio-init", "-c", script, capture=True)
|
||||
return int(output.splitlines()[-1])
|
||||
|
||||
|
||||
def wait_marker(attachment_id, column, seconds=60):
|
||||
wait_database(
|
||||
"SELECT " + column + "::text FROM attachment_jobs WHERE attachment_id='"
|
||||
+ attachment_id + "' ORDER BY pipeline_version DESC LIMIT 1;", "true", seconds)
|
||||
|
||||
|
||||
def wait_queue_empty(seconds=150):
|
||||
deadline = time.monotonic() + seconds
|
||||
marker = env_value("ATTACHMENT_JOB_QUEUE") + "\t0\t0"
|
||||
while time.monotonic() < deadline:
|
||||
listing = command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues",
|
||||
"name", "messages_ready", "messages_unacknowledged", capture=True)
|
||||
if marker in listing:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise Failure("main queue did not drain")
|
||||
|
||||
|
||||
def wait_worker_consumer(seconds=150):
|
||||
deadline = time.monotonic() + seconds
|
||||
queue = env_value("ATTACHMENT_JOB_QUEUE")
|
||||
while time.monotonic() < deadline:
|
||||
listing = command(
|
||||
"exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues",
|
||||
"name", "consumers", capture=True)
|
||||
for line in listing.splitlines():
|
||||
columns = line.split("\t")
|
||||
if len(columns) == 2 and columns[0] == queue and int(columns[1]) > 0:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure("worker did not register a RabbitMQ consumer")
|
||||
|
||||
|
||||
def assert_ready_invariants(attachment_id, expected_version):
|
||||
row = database_scalar(
|
||||
"SELECT pipeline_version || '|' || status FROM attachments WHERE id='"
|
||||
+ attachment_id + "';")
|
||||
check(row == str(expected_version) + "|ready", "active pipeline is not ready")
|
||||
check(database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" + attachment_id
|
||||
+ "' AND pipeline_version=" + str(expected_version) + ";") == "3",
|
||||
"active pipeline does not have exactly three variants")
|
||||
check(database_scalar(
|
||||
"SELECT count(*) FROM attachment_jobs WHERE attachment_id='" + attachment_id
|
||||
+ "' AND status IN ('queued','processing');") == "0",
|
||||
"acknowledged attachment still has non-terminal work")
|
||||
|
||||
|
||||
def fault_case(client, alice, bob, chat, flag, marker, expected_objects_before_commit):
|
||||
recreate(
|
||||
"worker", **{flag: "true", "JOB_TIMEOUT": "5s", "JOB_LEASE_DURATION": "8s"})
|
||||
wait_worker_consumer()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
wait_marker(attachment["id"], marker, seconds=150)
|
||||
check(database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='"
|
||||
+ attachment["id"] + "';") == "0",
|
||||
"result metadata committed before the injected pre-commit crash")
|
||||
check(object_count(attachment["id"]) == expected_objects_before_commit,
|
||||
"unexpected object count at injected fault point")
|
||||
ready = client.wait_ready(bob, attachment, wait_seconds=150)
|
||||
check(ready["status"] == "ready", "faulted job was lost")
|
||||
assert_ready_invariants(attachment["id"], 2)
|
||||
check(int(database_scalar(
|
||||
"SELECT attempt FROM attachment_jobs WHERE attachment_id='"
|
||||
+ attachment["id"] + "' AND pipeline_version=2;")) >= 2,
|
||||
"crashed job was not recovered through a later managed attempt")
|
||||
check(object_count(attachment["id"]) == 4,
|
||||
"idempotent retry did not converge to one deterministic object set")
|
||||
recreate("worker")
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def main():
|
||||
client = Client("http://localhost:8080")
|
||||
recreate("api", ATTACHMENT_PIPELINE_VERSION="1")
|
||||
wait_http(client)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
|
||||
# Complete v1, promote to v2, then deliver newer work before stale v1 and
|
||||
# duplicate v2 messages. Arrival order intentionally differs from creation order.
|
||||
versioned, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, versioned, wait_seconds=120)
|
||||
version1 = variant_snapshot(versioned["id"], 1)
|
||||
check(bool(version1), "v1 result is missing")
|
||||
command("stop", "worker")
|
||||
promote(versioned["id"], 2)
|
||||
publish(versioned["id"], 2)
|
||||
publish(versioned["id"], 2)
|
||||
publish(versioned["id"], 1)
|
||||
command("up", "-d", "--no-deps", "worker")
|
||||
client.wait_ready(bob, versioned, wait_seconds=120)
|
||||
wait_queue_empty()
|
||||
version2 = variant_snapshot(versioned["id"], 2)
|
||||
check(bool(version2) and version2 != version1,
|
||||
"v2 did not produce a distinct immutable versioned result")
|
||||
check(database_scalar(
|
||||
"SELECT count(*) FROM attachment_variants WHERE attachment_id='"
|
||||
+ versioned["id"] + "';") == "6", "versioned rows were lost or duplicated")
|
||||
stable = version2
|
||||
publish(versioned["id"], 1)
|
||||
publish(versioned["id"], 2)
|
||||
wait_queue_empty()
|
||||
check(variant_snapshot(versioned["id"], 2) == stable,
|
||||
"stale or duplicate delivery overwrote the active v2 result")
|
||||
assert_ready_invariants(versioned["id"], 2)
|
||||
|
||||
# Promote a still-queued v1 intent to v2. The old job must become
|
||||
# superseded, and its already-durable message must be a harmless no-op.
|
||||
command("stop", "worker")
|
||||
superseded, _ = client.attachment(alice, chat)
|
||||
check(job_status(superseded["id"], 1).startswith("queued|"),
|
||||
"new upload did not start as a queued v1 job")
|
||||
promote(superseded["id"], 2)
|
||||
check(job_status(superseded["id"], 1) == "superseded|true|superseded_by_v2",
|
||||
"queued v1 job was not fenced as superseded by v2")
|
||||
check(variant_count(superseded["id"], 1) == "0",
|
||||
"superseded job already has v1 result rows")
|
||||
publish(superseded["id"], 1)
|
||||
command("up", "-d", "--no-deps", "worker")
|
||||
client.wait_ready(bob, superseded, wait_seconds=120)
|
||||
wait_queue_empty()
|
||||
check(variant_count(superseded["id"], 1) == "0",
|
||||
"stale superseded v1 delivery created v1 result rows")
|
||||
check(variant_count(superseded["id"], 2) == "3",
|
||||
"promoted v2 job did not produce exactly three variants")
|
||||
assert_ready_invariants(superseded["id"], 2)
|
||||
|
||||
recreate("api", ATTACHMENT_PIPELINE_VERSION="2")
|
||||
wait_http(client)
|
||||
before_id = fault_case(
|
||||
client, alice, bob, chat, "FAULT_CRASH_BEFORE_SIDE_EFFECT",
|
||||
"before_effect_fault_injected", 0)
|
||||
after_id = fault_case(
|
||||
client, alice, bob, chat, "FAULT_CRASH_AFTER_SIDE_EFFECT",
|
||||
"after_effect_fault_injected", 4)
|
||||
|
||||
# Existing level-4 fault point proves redelivery after result COMMIT but before ack.
|
||||
recreate("worker", FAULT_CRASH_AFTER_RESULT_COMMIT="true")
|
||||
wait_worker_consumer()
|
||||
committed, _ = client.attachment(alice, chat)
|
||||
wait_marker(committed["id"], "result_fault_injected")
|
||||
client.wait_ready(bob, committed, wait_seconds=120)
|
||||
wait_queue_empty()
|
||||
assert_ready_invariants(committed["id"], 2)
|
||||
recreate("worker")
|
||||
|
||||
confirmed = database_scalar(
|
||||
"SELECT count(*) FROM attachments WHERE id IN ('" + versioned["id"] + "','"
|
||||
+ superseded["id"] + "','" + before_id + "','" + after_id + "','"
|
||||
+ committed["id"] + "') AND status='ready';")
|
||||
check(confirmed == "5", "a confirmed task was lost")
|
||||
print("PASS: version fencing, superseded, duplicate/reorder and "
|
||||
"pre/post-side-effect crashes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
run(main)
|
||||
finally:
|
||||
try:
|
||||
recreate("api", ATTACHMENT_PIPELINE_VERSION="2")
|
||||
recreate("worker")
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user