From cf9d9a918ffaf8c158de5c37fc7101587fa16eab Mon Sep 17 00:00:00 2001 From: David Date: Mon, 9 Nov 2026 01:23:49 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=BE=D0=BD=D0=B8=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D1=8B=D0=B9=20pipeline=20=D0=B8?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D1=83=20=D0=B8?= =?UTF-8?q?=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5=D0=BD=D1=82=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B8.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Трек 2 фиксирует дубль, устаревшую версию, падение до и после записи объекта и replay без второй копии результата. Co-authored-by: Cursor --- .github/pull_request_template.md | 12 +- REPORT.md | 20 +- app/attachments.py | 10 +- app/jobs.py | 268 ++++++++++++++++++++++----- app/main.py | 8 + app/pipeline.py | 26 ++- app/routes.py | 64 ++++++- evidence/track2.json | 40 ++++ migrations/006_pipeline_versions.sql | 14 ++ scripts/track2_experiment.py | 251 +++++++++++++++++++++++++ 10 files changed, 641 insertions(+), 72 deletions(-) create mode 100644 evidence/track2.json create mode 100644 migrations/006_pipeline_versions.sql create mode 100644 scripts/track2_experiment.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 486cf77c..00356348 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 Автор / группа: Булацев, БПИ2301 -Целевая оценка (3/4/5): 4 -Трек 5, если заявлен: нет +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 2, доказанная идемпотентность pipeline -Что реализовано и что перенесено: база lab4 (API ×2, Postgres, Redis, SeaweedFS, вложения). Upload переведён на 202: staging в S3, задание в Postgres, RabbitMQ, отдельный worker, thumbnail/crop/optimized, outbox между COMMIT и publish. +Что реализовано и что перенесено: база lab4 (API ×2, Postgres, Redis, SeaweedFS, вложения). Upload переведён на 202: staging в S3, задание в Postgres, RabbitMQ, отдельный worker, thumbnail/crop/optimized, outbox между COMMIT и publish. Трек 2: версии pipeline и автоматический fault-injection (дубль, устаревшая версия, crash до и после side effect, replay). Команды запуска из чистого clone: см. REPORT.md (`cp .env.example .env`, `docker compose --env-file .env up -d --build --scale api=2`). -Результаты проверок и ссылка на REPORT.md: smoke PASS, `scripts/lab5_checks.py` PASS, `scripts/grade4_checks.py` PASS, разрыв с остановленным RabbitMQ доходит до ready. Подробности в REPORT.md. -Ограничения / известные ошибки: трек 5 не делался. Crash до ack и повтор persistence/failover на этом стенде ещё не прогонялись. Пока lab4 занимает 8080, этот стенд поднимался на 8081. +Результаты проверок и ссылка на REPORT.md: smoke PASS, `scripts/lab5_checks.py` PASS, `scripts/grade4_checks.py` PASS, `scripts/track2_experiment.py` PASS. Сырые sha и id в `evidence/track2.json`. Подробности в REPORT.md. +Ограничения / известные ошибки: трек 1 не делался. Повтор persistence/failover lab3 на этом стенде не прогонялся. Пока lab4 занимает 8080, стенд поднимался на 8081. Источники и использование ИИ: контракт репозитория, документация RabbitMQ и Pillow. Код и прогон проверок — с помощью Cursor. - [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [x] Контракт и публичные тесты не ослаблены. - [x] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. - [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. - [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index 4997767c..208b3d86 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 5 - Автор, группа: Булацев, БПИ2301 -- Целевая оценка; выбранный трек 5, если нужен: 4. Трек 5 не заявляется. +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 2 «Доказанная идемпотентность pipeline». - Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/33 - Использованные источники, библиотеки и помощь ИИ: контракт репозитория, RabbitMQ reliability / publisher confirms, aio-pika, Pillow. Реализация и прогон проверок сделаны с помощью Cursor. @@ -15,7 +15,7 @@ cp .env.example .env docker compose --env-file .env up -d --build --scale api=2 ``` -`migrate` применяет SQL один раз, включая `005_jobs.sql`. `s3-init` создаёт приватный bucket. Брокер — RabbitMQ с томом `rabbitmq_data`. Очереди `attachment.jobs`, `attachment.jobs.retry` и `attachment.jobs.dlq` объявляет приложение при первом соединении. Снаружи опубликован только `127.0.0.1:8080` у HAProxy. Реплики API, worker, Postgres, Redis, RabbitMQ и SeaweedFS наружу не публикуются. Порт S3 `127.0.0.1:9000` оставлен для presigned PUT с хоста, как в lab4. +`migrate` применяет SQL один раз, включая `005_jobs.sql` и `006_pipeline_versions.sql`. `s3-init` создаёт приватный bucket. Брокер — RabbitMQ с томом `rabbitmq_data`. Очереди `attachment.jobs`, `attachment.jobs.retry` и `attachment.jobs.dlq` объявляет приложение при первом соединении. Снаружи опубликован только `127.0.0.1:8080` у HAProxy. Реплики API, worker, Postgres, Redis, RabbitMQ и SeaweedFS наружу не публикуются. Порт S3 `127.0.0.1:9000` оставлен для presigned PUT с хоста, как в lab4. Готовность: `GET http://127.0.0.1:8080/health/ready` отвечает 200, когда доступны Postgres, Redis и bucket. Недоступный брокер не переводит API в 503 и не заставляет API делать resize: задание остаётся в таблице `attachment_jobs` с `published_at IS NULL`. @@ -24,6 +24,7 @@ python -X utf8 scripts/check_contract.py python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 python -X utf8 scripts/lab5_checks.py python -X utf8 scripts/grade4_checks.py +python -X utf8 scripts/track2_experiment.py ``` Остановка без удаления данных: `docker compose --env-file .env stop`. Остановить только worker: `sh scripts/stop-worker.sh`. Полное удаление учебных volumes: `docker compose --env-file .env down -v`. @@ -38,7 +39,7 @@ python -X utf8 scripts/grade4_checks.py Публикация идёт из outbox: `published_at` ставится после publisher confirm. Повтор той же строки даёт то же `job_id`. Worker — отдельный процесс `python -m app.worker`. Он копирует оригинал в `originals/`, строит варианты и только потом делает ack. Пока статус не `ready`, `GET .../content` отвечает 409. `queued` и `processing` можно прикрепить к сообщению, `failed` — нельзя. -Обязательный thumbnail: JPEG quality 90, вписан в 256×256 без увеличения, размеры округляются вниз, EXIF orientation применяется, альфа кладётся на белый. Fixture 320×200 даёт 256×160. На уровне 4 вместе с ним пишутся crop (центрированный JPEG 128×128, маленькие кадры увеличиваются) и optimized (WebP quality 80, method 6, вписан в 1280×1280 без увеличения, EXIF не копируется). `ready` ставится одним COMMIT вместе со всеми тремя вариантами. Ключ эффекта — `(attachment_id, pipeline_version, name)`, pipeline_version=1. +Обязательный thumbnail: JPEG quality 90, вписан в 256×256 без увеличения, размеры округляются вниз, EXIF orientation применяется, альфа кладётся на белый. Fixture 320×200 даёт 256×160. На уровне 4 вместе с ним пишутся crop (центрированный JPEG 128×128, маленькие кадры увеличиваются) и optimized (WebP quality 80, method 6, вписан в 1280×1280 без увеличения, EXIF не копируется). `ready` ставится одним COMMIT вместе со всеми тремя вариантами. Ключ эффекта — `(attachment_id, pipeline_version, variant)`. Обязательный путь использует pipeline_version=1. Версия 2 и дальше — тот же кадр, но JPEG quality 60 и WebP quality 50, чтобы байты отличались от версии 1. Объекты лежат в `variants//v/`. Публичный ответ отдаёт только `applied_pipeline_version`: более старая job не меняет это поле и не переписывает уже применённые варианты. Повтор: до 5 попыток (`JOB_MAX_ATTEMPTS`). Временный сбой возвращает job в `queued` с backoff `0.5 * 2^(attempt-1)` секунд, не больше 30, и множителем 0.5–1.5. Постоянная ошибка картинки — `invalid_image` без повторов. Исчерпание попыток — `retry_exhausted`. Оба финальных кода пишутся в Postgres и лучшим усилием в DLQ. Повторная доставка уже `ready` job только проверяет строку и делает ack. Явный replay: `python -m app.replay JOB_ID` или `POST /api/v1/_lab/attachments/{id}/replay` при `APP_ENV=lab`. Для `ready` это no-op, `job_id` не меняется. @@ -64,7 +65,8 @@ Lease равен `JOB_TIMEOUT_SECONDS + 15`. Просроченный `processin | Upload при остановленном worker | `python -X utf8 scripts/lab5_checks.py` | worker остановлен, upload 202, content 409, текстовое сообщение с этим id — 201; после start worker вложение `aae0c472-a799-4d51-97af-f88e7dbb0b5f` стало ready, варианты thumbnail/crop/optimized | вывод команды | | Негативные файлы lab4 на новом коде | `python -X utf8 scripts/grade4_checks.py` | PASS: 202 queued, затем скачивание оригинала; 400/413/415/404 без 500 | вывод команды | | Разрыв COMMIT и publish | `docker compose stop rabbitmq`, upload, SQL, `start rabbitmq` | upload 202, `published_at` NULL, status queued; после старта брокера job `1c92a0a4-f823-4f94-b3d9-c10c30bf2e4c` стал ready, вариантов 3 | вывод `psql` | -| Ограниченный retry, lease, DLQ, replay, crash до ack | код в `app/jobs.py`, `app/worker.py`, `WORKER_CRASH_POINT=after_s3` | в этом прогоне отдельно не воспроизводились | — | +| Crash до side effect, crash после PUT до ack, replay failed | `python -X utf8 scripts/track2_experiment.py` | все три случая дошли до ready с одним набором из 3 вариантов; оригинал не изменился; повтор после replay не создал второй набор | `evidence/track2.json` | +| Трек 2, дубль и устаревшая версия | тот же скрипт | два redeliver версии 1 оставили 3 строки и тот же sha thumbnail; версия 2 сменила sha, повтор старого `job_id` не вернул версию 1 | `evidence/track2.json` | `scripts/restart.sh` перезапускает postgres, redis, s3, rabbitmq, api и worker и не удаляет volumes. `scripts/stop-one.sh` по-прежнему останавливает одну API-реплику. Повторный прогон persistence/failover на стенде lab5 в этом заходе не делался. @@ -74,11 +76,17 @@ Lease равен `JOB_TIMEOUT_SECONDS + 15`. Просроченный `processin | --- | --- | --- | --- | | Остановлен worker, затем upload | 202 и сохранённое задание, content 409, чат жив | так и было; после start worker тот же id стал ready | около 1 с после start, файл не потерян | | Остановлен RabbitMQ после COMMIT | 202, `published_at` пустой; после старта брокера задача доходит до ready | SQL показал unpublished=t, затем ready и 3 варианта | около 19 с после `start rabbitmq`, новый upload не создавался | +| Дубль готовой job | ack без второго набора вариантов | sha thumbnail не изменился, строк вариантов 3 | эффект не потерян и не удвоен | +| Новая версия, затем старый job_id | обслуживается версия 2 | sha сменился с `fa0db661…` на `0ecf36b5…`, applied_pipeline_version остался 2 | старые 3 строки версии 1 остались историей, отдача не откатилась | +| `crash_before_s3` и `crash_after_s3` | процесс worker падает, Docker поднимает его снова, lease возвращает ту же job | оба вложения стали ready, оригинал совпал с fixture, вариантов ровно 3 | около 75 с на lease, байты оригинала не потеряны | +| `fail_once`, затем replay и ещё один дубль | терминальный failed, затем тот же job_id снова ready | error_code был `processing_failed`, после replay status ready и снова 3 варианта | подтверждённый результат не размножился | -Трек 5 не выполнялся. Отдельного замера throughput/RSS очереди нет. +Гипотеза трека 2: at-least-once и ключ `(attachment_id, pipeline_version, variant)` сохраняют самый новый зафиксированный pipeline и не теряют уже подтверждённую задачу. Методика повторяема: `scripts/track2_experiment.py` сам включает fault на одну job, ждёт ready и сверяет sha. Это не exactly-once: брокер присылает то же сообщение снова, а падение после PUT и до ack повторяет запись объекта. Один раз фиксируется COMMIT версии. Старая версия не затирает более новую. + +Трек 1 (очередь под перегрузкой) не выбирался. ## Ограничения и следующие шаги Стенд выдерживает остановку worker и разрыв между COMMIT и подтверждением брокера, пока живы Postgres и том RabbitMQ. Он не выдерживает потерю тома Postgres, тома S3 или единственного Docker-host. Потеря тома RabbitMQ до confirm компенсируется outbox в Postgres; потеря staging-объекта до копирования в `originals/` делает job невыполнимой. -Не сделано в этом прогоне: crash worker после PUT и до ack, явный replay failed, persistence/failover скрипты lab3 на новом стенде, трек 5. `WORKER_CRASH_POINT=after_s3` завершает процесс после записи объектов и до COMMIT; повторная доставка должна сойтись в те же ключи. Это ещё нужно показать на защите. +Не повторялся на этом стенде прогон persistence/failover скриптами lab3. Потеря тома Postgres, тома S3 или единственного Docker-host не переживается. Потеря тома RabbitMQ до confirm компенсируется outbox в Postgres; потеря staging-объекта до копирования в `originals/` делает job невыполнимой. Fault-injection трека 2 включается только при `APP_ENV=lab`. diff --git a/app/attachments.py b/app/attachments.py index 499bbe4c..b2adee78 100644 --- a/app/attachments.py +++ b/app/attachments.py @@ -34,13 +34,13 @@ def _public(row, variants=None) -> dict: } -async def _variants(attachment_id) -> list: +async def _variants(attachment_id, pipeline_version: int) -> list: async with db.require().acquire() as conn: rows = await conn.fetch( """ SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants - WHERE attachment_id = $1 AND pipeline_version = 1 + WHERE attachment_id = $1 AND pipeline_version = $2 ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 @@ -49,6 +49,7 @@ async def _variants(attachment_id) -> list: END """, attachment_id, + pipeline_version, ) return [ { @@ -64,7 +65,8 @@ async def _variants(attachment_id) -> list: async def present(row) -> dict: - variants = await _variants(row["id"]) if row["status"] == "ready" else [] + version = int(row["applied_pipeline_version"]) if "applied_pipeline_version" in row.keys() else 1 + variants = await _variants(row["id"], version) if row["status"] == "ready" else [] return _public(row, variants) @@ -175,7 +177,7 @@ async def get_for_member(attachment_id: str, user_id: str): """ SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, a.status, a.error_code, - a.created_at, a.updated_at + a.applied_pipeline_version, a.created_at, a.updated_at FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1 diff --git a/app/jobs.py b/app/jobs.py index 52240139..a9dc730d 100644 --- a/app/jobs.py +++ b/app/jobs.py @@ -155,10 +155,11 @@ async def claim(job_id: UUID, pipeline_version: int): if row["status"] == "ready": return "ready", row attachment = await conn.fetchrow( - "SELECT status FROM attachments WHERE id = $1", + "SELECT status, applied_pipeline_version FROM attachments WHERE id = $1", row["attachment_id"], ) - if attachment is not None and attachment["status"] == "ready": + applied = int(attachment["applied_pipeline_version"]) if attachment is not None else 0 + if applied > int(row["pipeline_version"]): await conn.execute( """ UPDATE attachment_jobs @@ -338,15 +339,22 @@ async def _intent(key: str) -> None: ) -async def complete_job(job_id: UUID, lease_token: UUID, attachment_id: UUID, final_key: str, variants: list) -> bool: +async def complete_job( + job_id: UUID, + lease_token: UUID, + attachment_id: UUID, + final_key: str, + variants: list, + pipeline_version: int, +) -> bool: async with db.require().acquire() as conn: async with conn.transaction(): - owned = await conn.fetchval( + owned = await conn.fetchrow( """ UPDATE attachment_jobs SET status = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = now() WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' - RETURNING job_id + RETURNING pipeline_version """, job_id, lease_token, @@ -354,55 +362,71 @@ async def complete_job(job_id: UUID, lease_token: UUID, attachment_id: UUID, fin if owned is None: return False current = await conn.fetchrow( - "SELECT status FROM attachments WHERE id = $1 FOR UPDATE", + """ + SELECT status, applied_pipeline_version + FROM attachments + WHERE id = $1 + FOR UPDATE + """, attachment_id, ) if current is None: raise PermanentJobError("invalid_image") - if current["status"] != "ready": + version = int(owned["pipeline_version"]) + applied = int(current["applied_pipeline_version"]) + superseded = applied > version + if not superseded: await conn.execute( """ UPDATE attachments - SET status = 'ready', object_key = $2, error_code = NULL, updated_at = now() - WHERE id = $1 AND status <> 'ready' + SET status = 'ready', + object_key = CASE WHEN $2 = '' THEN object_key ELSE $2 END, + error_code = NULL, + applied_pipeline_version = $3, + updated_at = now() + WHERE id = $1 AND applied_pipeline_version <= $3 """, attachment_id, final_key, + version, ) - for variant in variants: - await conn.execute( - """ - INSERT INTO attachment_variants ( - attachment_id, pipeline_version, name, content_type, size_bytes, - width, height, sha256, object_key + for variant in variants: + await conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + attachment_id, + version, + variant["name"], + variant["content_type"], + variant["size_bytes"], + variant["width"], + variant["height"], + variant["sha256"], + variant["object_key"], + ) + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + variant["object_key"], + ) + if final_key: + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + final_key, ) - VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING - """, - attachment_id, - variant["name"], - variant["content_type"], - variant["size_bytes"], - variant["width"], - variant["height"], - variant["sha256"], - variant["object_key"], - ) - await conn.execute( - "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", - variant["object_key"], - ) - await conn.execute( - "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", - final_key, - ) await conn.execute( """ INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) - VALUES ($1, $2, 'ready', 'variants committed') + VALUES ($1, $2, 'ready', $3) """, job_id, attachment_id, + "superseded" if superseded else "variants committed", ) return True @@ -411,7 +435,8 @@ async def load_attachment(attachment_id: UUID): async with db.require().acquire() as conn: return await conn.fetchrow( """ - SELECT id, filename, content_type, size_bytes, sha256, object_key, status + SELECT id, filename, content_type, size_bytes, sha256, object_key, status, + applied_pipeline_version FROM attachments WHERE id = $1 """, @@ -442,11 +467,18 @@ async def _effect(row, lease: UUID) -> None: attachment = await load_attachment(row["attachment_id"]) if attachment is None: raise PermanentJobError("invalid_image") - if attachment["status"] == "ready": - committed = await complete_job(row["job_id"], lease, attachment["id"], attachment["object_key"], []) + version = int(row["pipeline_version"]) + if int(attachment["applied_pipeline_version"]) > version: + committed = await complete_job(row["job_id"], lease, attachment["id"], "", [], version) if not committed: raise TransientJobError("processing_failed") return + fault = await consume_fault() + if fault == "crash_before_s3": + logger.warning("injected crash before side effect job_id=%s", row["job_id"]) + os._exit(1) + if fault == "fail_once": + raise PermanentJobError("processing_failed") data = await s3.get_bytes(attachment["object_key"]) if len(data) != attachment["size_bytes"] or hashlib.sha256(data).hexdigest() != attachment["sha256"]: raise PermanentJobError("invalid_image") @@ -454,11 +486,11 @@ async def _effect(row, lease: UUID) -> None: if attachment["object_key"] != final_key: await _intent(final_key) await s3.put_bytes(final_key, data, attachment["content_type"]) - rendered = await asyncio.to_thread(pipeline.render, data, attachment["content_type"]) + rendered = await asyncio.to_thread(pipeline.render, data, attachment["content_type"], version) stored = [] for name, (payload, content_type, width, height) in rendered.items(): suffix = "webp" if content_type == "image/webp" else "jpg" - key = f"variants/{attachment['id']}/{name}.{suffix}" + key = f"variants/{attachment['id']}/v{version}/{name}.{suffix}" await _intent(key) await s3.put_bytes(key, payload, content_type) stored.append( @@ -472,13 +504,17 @@ async def _effect(row, lease: UUID) -> None: "object_key": key, } ) - if config.WORKER_CRASH_POINT == "after_s3": + if fault == "crash_after_s3" or config.WORKER_CRASH_POINT == "after_s3": logger.warning("injected crash after object write before result commit job_id=%s", row["job_id"]) os._exit(1) - committed = await complete_job(row["job_id"], lease, attachment["id"], final_key, stored) + fresh = await load_attachment(attachment["id"]) + if fresh is not None and int(fresh["applied_pipeline_version"]) > version: + stored = [] + final_key = "" + committed = await complete_job(row["job_id"], lease, attachment["id"], final_key, stored, version) if not committed: raise TransientJobError("processing_failed") - if attachment["object_key"] != final_key: + if stored and attachment["object_key"] != final_key and final_key: try: await s3.delete_key(attachment["object_key"]) async with db.require().acquire() as conn: @@ -487,12 +523,154 @@ async def _effect(row, lease: UUID) -> None: logger.warning("staging object left for retention attachment_id=%s", attachment["id"]) +async def consume_fault() -> str: + async with db.require().acquire() as conn: + row = await conn.fetchrow( + """ + UPDATE pipeline_faults + SET remaining = remaining - 1 + WHERE id = 1 AND remaining > 0 + RETURNING mode + """ + ) + if row is None: + return "" + return row["mode"] or "" + + +async def arm_fault(mode: str, remaining: int) -> dict: + if mode not in ("", "crash_before_s3", "crash_after_s3", "fail_once"): + raise AppError(400, "invalid fault") + if remaining < 0 or remaining > 5: + raise AppError(400, "invalid fault") + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO pipeline_faults (id, mode, remaining) + VALUES (1, $1, $2) + ON CONFLICT (id) DO UPDATE SET mode = EXCLUDED.mode, remaining = EXCLUDED.remaining + """, + mode, + remaining, + ) + return {"mode": mode, "remaining": remaining} + + +async def bump_pipeline(attachment_id: UUID) -> dict: + created = datetime.now(timezone.utc) + async with db.require().acquire() as conn: + async with conn.transaction(): + current = await conn.fetchval( + "SELECT COALESCE(MAX(pipeline_version), 0) FROM attachment_jobs WHERE attachment_id = $1", + attachment_id, + ) + if current is None: + return {"result": "missing"} + exists = await conn.fetchval("SELECT 1 FROM attachments WHERE id = $1", attachment_id) + if exists is None: + return {"result": "missing"} + version = int(current) + 1 + job_id = uuid4() + await conn.execute( + """ + INSERT INTO attachment_jobs ( + job_id, attachment_id, pipeline_version, attempt, status, + next_attempt_at, created_at, updated_at + ) + VALUES ($1, $2, $3, 1, 'queued', $4, $4, $4) + """, + job_id, + attachment_id, + version, + created, + ) + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'queued', 'pipeline version bumped') + """, + job_id, + attachment_id, + ) + kick_publish() + return {"result": "queued", "job_id": str(job_id), "pipeline_version": version} + + +async def redeliver(job_id: UUID) -> dict: + async with db.require().acquire() as conn: + row = await conn.fetchrow( + """ + SELECT job_id, attachment_id, pipeline_version, attempt, created_at, status + FROM attachment_jobs + WHERE job_id = $1 + """, + job_id, + ) + if row is None: + return {"result": "missing"} + await broker.publish_job(message_of(row)) + return { + "result": "published", + "job_id": str(row["job_id"]), + "pipeline_version": int(row["pipeline_version"]), + "status": row["status"], + } + + +async def list_jobs(attachment_id: UUID) -> list: + async with db.require().acquire() as conn: + rows = await conn.fetch( + """ + SELECT job_id, pipeline_version, status, attempt + FROM attachment_jobs + WHERE attachment_id = $1 + ORDER BY pipeline_version + """, + attachment_id, + ) + variants = await conn.fetch( + """ + SELECT pipeline_version, name, sha256 + FROM attachment_variants + WHERE attachment_id = $1 + ORDER BY pipeline_version, name + """, + attachment_id, + ) + applied = await conn.fetchval( + "SELECT applied_pipeline_version FROM attachments WHERE id = $1", + attachment_id, + ) + return { + "applied_pipeline_version": applied, + "jobs": [ + { + "job_id": str(row["job_id"]), + "pipeline_version": int(row["pipeline_version"]), + "status": row["status"], + "attempt": int(row["attempt"]), + } + for row in rows + ], + "variants": [ + { + "pipeline_version": int(row["pipeline_version"]), + "name": row["name"], + "sha256": row["sha256"], + } + for row in variants + ], + } + + async def replay_attachment(attachment_id: str) -> dict: async with db.require().acquire() as conn: job_id = await conn.fetchval( """ SELECT job_id FROM attachment_jobs - WHERE attachment_id = $1 AND pipeline_version = 1 + WHERE attachment_id = $1 AND status = 'failed' + ORDER BY pipeline_version DESC + LIMIT 1 """, UUID(attachment_id), ) diff --git a/app/main.py b/app/main.py index 9a450e2f..c06bdf59 100644 --- a/app/main.py +++ b/app/main.py @@ -28,8 +28,12 @@ from app.routes import ( get_chat, health_live, health_ready, + lab_bump, + lab_fault, lab_flags, + lab_jobs, lab_metrics, + lab_redeliver, lab_replay, lab_sweep, list_chats, @@ -105,6 +109,10 @@ def build_app() -> Starlette: Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]), Route("/api/v1/_lab/sweep", lab_sweep, methods=["POST"]), + Route("/api/v1/_lab/faults", lab_fault, methods=["POST"]), + Route("/api/v1/_lab/attachments/{attachment_id}/jobs", lab_jobs, methods=["GET"]), + Route("/api/v1/_lab/attachments/{attachment_id}/pipelines", lab_bump, methods=["POST"]), + Route("/api/v1/_lab/jobs/{job_id}/redeliver", lab_redeliver, methods=["POST"]), Route("/api/v1/_lab/attachments/{attachment_id}/replay", lab_replay, methods=["POST"]), ], exception_handlers={404: not_found_handler, AppError: app_error_handler}, diff --git a/app/pipeline.py b/app/pipeline.py index e05c01da..af184199 100644 --- a/app/pipeline.py +++ b/app/pipeline.py @@ -69,27 +69,33 @@ def _crop_square(image: Image.Image, side: int) -> Image.Image: return cropped -def _jpeg(image: Image.Image) -> bytes: +def _jpeg(image: Image.Image, quality: int) -> bytes: buffer = io.BytesIO() - image.save(buffer, format="JPEG", quality=90, optimize=True) + image.save(buffer, format="JPEG", quality=quality, optimize=True) return buffer.getvalue() -def _webp(image: Image.Image) -> bytes: - # quality=80, method=6. EXIF is omitted because the saved image is a new RGB buffer. +def _webp(image: Image.Image, quality: int) -> bytes: + # method=6. EXIF is omitted because the saved image is a new RGB buffer. buffer = io.BytesIO() - image.save(buffer, format="WEBP", quality=80, method=6) + image.save(buffer, format="WEBP", quality=quality, method=6) return buffer.getvalue() -def render(data: bytes, content_type: str) -> dict: - """Return thumbnail, crop and optimized payloads. Does not touch storage.""" +def render(data: bytes, content_type: str, pipeline_version: int = 1) -> dict: + """Return thumbnail, crop and optimized payloads. Does not touch storage. + + Version 1 is the contract profile (JPEG 90, WebP 80). A later version lowers + quality so a stale writer cannot be confused with the newer bytes. + """ + jpeg_quality = 90 if pipeline_version <= 1 else 60 + webp_quality = 80 if pipeline_version <= 1 else 50 image = _open_rgb(data, content_type) thumb = _fit_down(image, 256) crop = _crop_square(image, 128) optimized = _fit_down(image, 1280) return { - "thumbnail": ( _jpeg(thumb), "image/jpeg", thumb.size[0], thumb.size[1]), - "crop": (_jpeg(crop), "image/jpeg", crop.size[0], crop.size[1]), - "optimized": (_webp(optimized), "image/webp", optimized.size[0], optimized.size[1]), + "thumbnail": (_jpeg(thumb, jpeg_quality), "image/jpeg", thumb.size[0], thumb.size[1]), + "crop": (_jpeg(crop, jpeg_quality), "image/jpeg", crop.size[0], crop.size[1]), + "optimized": (_webp(optimized, webp_quality), "image/webp", optimized.size[0], optimized.size[1]), } diff --git a/app/routes.py b/app/routes.py index 76d23435..735a67a4 100644 --- a/app/routes.py +++ b/app/routes.py @@ -2,6 +2,8 @@ import base64 import binascii import re +from uuid import UUID + from starlette.requests import Request from starlette.responses import JSONResponse, Response @@ -398,9 +400,10 @@ async def download_attachment(request: Request) -> Response: """ SELECT content_type, object_key FROM attachment_variants - WHERE attachment_id = $1 AND pipeline_version = 1 AND name = $2 + WHERE attachment_id = $1 AND pipeline_version = $2 AND name = $3 """, row["id"], + int(row["applied_pipeline_version"]), variant, ) if found is None: @@ -410,6 +413,65 @@ async def download_attachment(request: Request) -> Response: return Response(payload, status_code=200, media_type=media_type, headers={"X-Request-Id": rid}) +async def lab_fault(request: Request) -> Response: + rid = request.state.request_id + if not config.lab_mode(): + return error_response(404, "not found", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + mode = body.get("mode", "") + remaining = body.get("remaining", 1) + if isinstance(remaining, bool) or not isinstance(remaining, int): + return error_response(400, "invalid fault", rid) + try: + armed = await jobs.arm_fault(mode, remaining) + except AppError as exc: + return error_response(exc.status, exc.message, rid) + return _json(armed, rid) + + +async def lab_jobs(request: Request) -> Response: + rid = request.state.request_id + if not config.lab_mode(): + return error_response(404, "not found", rid) + attachment_id = request.path_params["attachment_id"] + if validate_uuid(attachment_id): + return error_response(400, "invalid attachment id", rid) + listed = await jobs.list_jobs(UUID(attachment_id)) + if listed["applied_pipeline_version"] is None: + return error_response(404, "attachment not found", rid) + return _json(listed, rid) + + +async def lab_bump(request: Request) -> Response: + rid = request.state.request_id + if not config.lab_mode(): + return error_response(404, "not found", rid) + attachment_id = request.path_params["attachment_id"] + if validate_uuid(attachment_id): + return error_response(400, "invalid attachment id", rid) + result = await jobs.bump_pipeline(UUID(attachment_id)) + if result["result"] == "missing": + return error_response(404, "attachment not found", rid) + return _json(result, rid, 202) + + +async def lab_redeliver(request: Request) -> Response: + rid = request.state.request_id + if not config.lab_mode(): + return error_response(404, "not found", rid) + job_id = request.path_params["job_id"] + if validate_uuid(job_id): + return error_response(400, "invalid job id", rid) + result = await jobs.redeliver(UUID(job_id)) + if result["result"] == "missing": + return error_response(404, "attachment not found", rid) + return _json(result, rid, 202) + + async def lab_replay(request: Request) -> Response: rid = request.state.request_id if not config.lab_mode(): diff --git a/evidence/track2.json b/evidence/track2.json new file mode 100644 index 00000000..c84c9d27 --- /dev/null +++ b/evidence/track2.json @@ -0,0 +1,40 @@ +{ + "hypothesis": "at-least-once plus a versioned effect key keeps the newest committed pipeline", + "cases": [ + { + "name": "duplicate", + "attachment_id": "31cbcb54-ef7e-4eb2-afb0-80024908df9f", + "thumbnail_sha256": "fa0db661078d4da91b241edc8b416de7755db9c9c79ae704a620ca7ad534e308", + "variant_rows": 3 + }, + { + "name": "reorder_and_stale_version", + "attachment_id": "31cbcb54-ef7e-4eb2-afb0-80024908df9f", + "version1_sha256": "fa0db661078d4da91b241edc8b416de7755db9c9c79ae704a620ca7ad534e308", + "version2_sha256": "0ecf36b5c16164108028abe72ac79d5a8671bebb5fd107cc85c5bf1311456985", + "applied_pipeline_version": 2, + "variant_rows": 6 + }, + { + "name": "crash_before_s3", + "attachment_id": "136bc3ad-78e9-47fa-9ee1-ae552ca3277c", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "variant_rows": 3, + "thumbnail_sha256": "fa0db661078d4da91b241edc8b416de7755db9c9c79ae704a620ca7ad534e308" + }, + { + "name": "crash_after_s3", + "attachment_id": "b4783cc2-3b1b-49a3-bb76-20c2d31e1eda", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "variant_rows": 3, + "thumbnail_sha256": "fa0db661078d4da91b241edc8b416de7755db9c9c79ae704a620ca7ad534e308" + }, + { + "name": "replay_then_duplicate", + "attachment_id": "652b1c3f-b70a-4266-a064-89a820fcfe35", + "variant_rows": 3, + "job_status": "ready" + } + ], + "not_exactly_once": "The broker delivers a message more than once. A crash after the object PUT and before ack runs the S3 write again. The database commit of a pipeline version happens once, and an older version cannot replace the applied one. That is idempotent effect, not exactly-once delivery." +} diff --git a/migrations/006_pipeline_versions.sql b/migrations/006_pipeline_versions.sql new file mode 100644 index 00000000..6e02f18a --- /dev/null +++ b/migrations/006_pipeline_versions.sql @@ -0,0 +1,14 @@ +-- The served result is one pipeline version. An older job must not replace it. +-- Variant rows stay addressable per version; the attachment points at the winner. + +ALTER TABLE attachments + ADD COLUMN applied_pipeline_version INTEGER NOT NULL DEFAULT 1 + CHECK (applied_pipeline_version >= 1); + +CREATE TABLE pipeline_faults ( + id INTEGER PRIMARY KEY, + mode TEXT NOT NULL DEFAULT '', + remaining INTEGER NOT NULL DEFAULT 0 CHECK (remaining >= 0) +); + +INSERT INTO pipeline_faults (id, mode, remaining) VALUES (1, '', 0); diff --git a/scripts/track2_experiment.py b/scripts/track2_experiment.py new file mode 100644 index 00000000..88d391a5 --- /dev/null +++ b/scripts/track2_experiment.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Track 2: versioned pipeline under duplicate, reorder, crash and a stale job. + +Prints statuses and attachment ids only. Tokens stay in memory. +""" + +import base64 +import hashlib +import json +import os +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" / "track2.json" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +FIXTURE = ROOT / "tests" / "fixtures" / "sample.png" + + +def compose(*args: str) -> None: + command = ["docker", "compose", "--env-file", ".env"] + extra = os.environ.get("LAB5_COMPOSE_FILES", "") + for name in extra.split(os.pathsep): + if name: + command.extend(["-f", name]) + subprocess.run([*command, *args], cwd=ROOT, check=True) + + +class Api: + def __init__(self) -> None: + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None): + hdrs = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + hdrs["Content-Type"] = "application/json" + if content_type: + hdrs["Content-Type"] = content_type + if token: + hdrs["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method) + try: + response = self.opener.open(request, timeout=30) + except urllib.error.HTTPError as error: + response = error + with response: + payload = response.read() + parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None + return response.status, parsed, payload + + +def expect(status, got, label): + if got != status: + raise SystemExit("FAIL %s expected %s got %s" % (label, status, got)) + + +def wait_applied(api: Api, attachment_id: str, token: str, version: int, seconds: int = 150, allow_failed: bool = False): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) + expect(200, status, "poll") + state = jobs(api, attachment_id) + if body["status"] == "failed" and allow_failed: + return body + if ( + body["status"] == "ready" + and state["applied_pipeline_version"] == version + and any(item["name"] == "thumbnail" for item in body["variants"]) + ): + return body + time.sleep(0.4) + raise SystemExit("FAIL timeout waiting for pipeline %s" % version) + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) + expect(200, status, "poll") + last = body + if body["status"] == "ready" and any(item["name"] == "thumbnail" for item in body["variants"]): + return body + if body["status"] == "failed": + return body + time.sleep(0.4) + raise SystemExit("FAIL timeout last=%s" % (last or {}).get("status")) + + +def jobs(api: Api, attachment_id: str): + status, body, _ = api.call("GET", "/api/v1/_lab/attachments/%s/jobs" % attachment_id) + expect(200, status, "jobs") + return body + + +def thumb(body): + found = [item for item in body["variants"] if item["name"] == "thumbnail"] + if len(found) != 1: + raise SystemExit("FAIL thumbnail count %s" % len(found)) + return found[0]["sha256"] + + +def user(api: Api): + username = "t2_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + status, body, _ = api.call( + "POST", + "/api/v1/users", + {"username": username, "display_name": "Track2", "password": password}, + ) + expect(201, status, "register") + basic = base64.b64encode((username + ":" + password).encode()).decode() + status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + expect(201, status, "login") + return body["id"], session["token"] + + +def upload(api: Api, chat_id: str, token: str, filename: str): + status, body, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename), + token=token, + raw=FIXTURE.read_bytes(), + content_type="image/png", + ) + expect(202, status, "upload " + filename) + return body + + +def main() -> None: + api = Api() + content = FIXTURE.read_bytes() + original = hashlib.sha256(content).hexdigest() + alice, token = user(api) + status, chat, _ = api.call( + "POST", + "/api/v1/chats", + {"title": "track2", "member_ids": [alice]}, + token=token, + ) + expect(201, status, "chat") + chat_id = chat["id"] + report = {"hypothesis": "at-least-once plus a versioned effect key keeps the newest committed pipeline", "cases": []} + + first = upload(api, chat_id, token, "v1.png") + ready = wait_applied(api, first["id"], token, 1) + sha_v1 = thumb(ready) + state = jobs(api, first["id"]) + v1_job = state["jobs"][0]["job_id"] + expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 1") + expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 2") + time.sleep(2) + again = wait_applied(api, first["id"], token, 1) + state = jobs(api, first["id"]) + v1_rows = [item for item in state["variants"] if item["pipeline_version"] == 1] + if thumb(again) != sha_v1 or len(v1_rows) != 3: + raise SystemExit("FAIL duplicate changed the effect") + report["cases"].append({"name": "duplicate", "attachment_id": first["id"], "thumbnail_sha256": sha_v1, "variant_rows": len(v1_rows)}) + + expect(202, api.call("POST", "/api/v1/_lab/attachments/%s/pipelines" % first["id"])[0], "bump") + bumped = wait_applied(api, first["id"], token, 2) + sha_v2 = thumb(bumped) + state = jobs(api, first["id"]) + if sha_v2 == sha_v1 or state["applied_pipeline_version"] != 2: + raise SystemExit("FAIL version 2 did not replace the served bytes") + v1_job = [item["job_id"] for item in state["jobs"] if item["pipeline_version"] == 1][0] + expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "stale redelivery") + time.sleep(2) + after_stale = wait_applied(api, first["id"], token, 2) + state = jobs(api, first["id"]) + if thumb(after_stale) != sha_v2 or state["applied_pipeline_version"] != 2: + raise SystemExit("FAIL stale job overwrote version 2") + report["cases"].append( + { + "name": "reorder_and_stale_version", + "attachment_id": first["id"], + "version1_sha256": sha_v1, + "version2_sha256": sha_v2, + "applied_pipeline_version": state["applied_pipeline_version"], + "variant_rows": len(state["variants"]), + } + ) + + def crashed(mode: str): + expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": mode, "remaining": 1})[0], mode) + created = upload(api, chat_id, token, mode + ".png") + # The worker process exits. Docker restarts it; the lease must expire before retry. + body = wait_applied(api, created["id"], token, 1, seconds=180) + if body["status"] != "ready": + raise SystemExit("FAIL %s ended %s" % (mode, body["status"])) + status_code, _, payload = api.call("GET", "/api/v1/attachments/%s/content" % created["id"], token=token) + expect(200, status_code, mode + " original") + if hashlib.sha256(payload).hexdigest() != original: + raise SystemExit("FAIL %s lost the original" % mode) + state_now = jobs(api, created["id"]) + rows = [item for item in state_now["variants"] if item["pipeline_version"] == 1] + if len(rows) != 3: + raise SystemExit("FAIL %s variant rows %s" % (mode, len(rows))) + report["cases"].append( + { + "name": mode, + "attachment_id": created["id"], + "original_sha256": original, + "variant_rows": len(rows), + "thumbnail_sha256": thumb(body), + } + ) + + crashed("crash_before_s3") + crashed("crash_after_s3") + + expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": "fail_once", "remaining": 1})[0], "fail_once") + failed = upload(api, chat_id, token, "fail.png") + failed_body = wait_applied(api, failed["id"], token, 1, allow_failed=True) + if failed_body["status"] != "failed" or failed_body["error_code"] != "processing_failed": + raise SystemExit("FAIL expected a terminal processing_failed") + expect(200, api.call("POST", "/api/v1/_lab/attachments/%s/replay" % failed["id"])[0], "replay") + restored = wait_applied(api, failed["id"], token, 1) + if restored["status"] != "ready": + raise SystemExit("FAIL replay did not reach ready") + state = jobs(api, failed["id"]) + replay_job = state["jobs"][0]["job_id"] + expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % replay_job)[0], "replay duplicate") + time.sleep(2) + state = jobs(api, failed["id"]) + if len(state["variants"]) != 3 or state["jobs"][0]["status"] != "ready": + raise SystemExit("FAIL replay produced another effect") + report["cases"].append( + { + "name": "replay_then_duplicate", + "attachment_id": failed["id"], + "variant_rows": len(state["variants"]), + "job_status": state["jobs"][0]["status"], + } + ) + + report["not_exactly_once"] = ( + "The broker delivers a message more than once. A crash after the object PUT and before ack " + "runs the S3 write again. The database commit of a pipeline version happens once, and an older " + "version cannot replace the applied one. That is idempotent effect, not exactly-once delivery." + ) + OUT.parent.mkdir(exist_ok=True) + OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS track2", OUT) + + +if __name__ == "__main__": + main()