From ab9161be3d99ec12aee9d6363ac50cfdb578c105 Mon Sep 17 00:00:00 2001 From: angelika005 Date: Fri, 13 Nov 2026 13:10:15 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=205=20=E2=80=94=20=D0=91=D0=92?= =?UTF-8?q?=D0=A22303=20=E2=80=94=20=D0=91=D1=80=D0=B5=D0=B7=D0=B3=D1=83?= =?UTF-8?q?=D0=BD=D0=BE=D0=B2=D0=B0=20=E2=80=94=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 5 + .github/pull_request_template.md | 34 +- .github/workflows/solution.yml | 28 + DEFENSE.md | 20 + Dockerfile | 15 + REPORT.md | 174 +- compose.test.yaml | 16 + compose.yaml | 181 ++ contracts/extensions.md | 17 + evidence/acceptance.txt | 1 + evidence/after-restart.json | 65 + evidence/before-restart.json | 65 + evidence/check.txt | 1 + evidence/docker-environment.json | 7 + evidence/failover.txt | 3 + evidence/final-containers.json | 65 + evidence/final-smoke.txt | 1 + evidence/host-cpu.json | 5 + evidence/host-memory.json | 3 + evidence/integrity.json | 8 + evidence/openapi-validation.txt | 1 + evidence/persistence.txt | 2 + evidence/pipeline-results.json | 2363 +++++++++++++++++++++++ evidence/post-commit-redelivery.json | 119 ++ evidence/production-containers.json | 65 + evidence/production-smoke.txt | 1 + evidence/retention-active-recovery.json | 99 + evidence/smoke.txt | 1 + evidence/staging-retention.txt | 1 + evidence/verification.json | 14 + infra/Dockerfile.minio | 19 + infra/api-policy.json | 1 + infra/bootstrap.sh | 16 + infra/broker-start.sh | 15 + infra/haproxy.cfg | 26 + infra/minio-start.sh | 5 + infra/operator-policy.json | 1 + infra/worker-policy.json | 1 + migrations/001_initial.sql | 39 + migrations/002_pipeline.sql | 28 + requirements.lock | 27 + requirements.txt | 7 + scripts/integrity.py | 30 + scripts/migrate.py | 8 + scripts/restart.cmd | 4 + scripts/restart.sh | 4 + scripts/setup.py | 25 + scripts/stop-one.cmd | 5 + scripts/stop-one.sh | 7 + scripts/verify.py | 251 +++ scripts/verify_cleanup.py | 16 + scripts/verify_redelivery.py | 21 + src/__init__.py | 1 + src/app.py | 523 +++++ src/cleanup.py | 39 + src/jobs.py | 78 + src/operator.py | 58 + src/publisher.py | 64 + src/storage.py | 30 + src/transform.py | 56 + src/worker.py | 223 +++ student-tests/dbtool.py | 64 + student-tests/maintenance.py | 45 + student-tests/pipeline.py | 108 ++ 64 files changed, 5191 insertions(+), 34 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/solution.yml create mode 100644 DEFENSE.md create mode 100644 Dockerfile create mode 100644 compose.test.yaml create mode 100644 compose.yaml create mode 100644 contracts/extensions.md create mode 100644 evidence/acceptance.txt create mode 100644 evidence/after-restart.json create mode 100644 evidence/before-restart.json create mode 100644 evidence/check.txt create mode 100644 evidence/docker-environment.json create mode 100644 evidence/failover.txt create mode 100644 evidence/final-containers.json create mode 100644 evidence/final-smoke.txt create mode 100644 evidence/host-cpu.json create mode 100644 evidence/host-memory.json create mode 100644 evidence/integrity.json create mode 100644 evidence/openapi-validation.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/pipeline-results.json create mode 100644 evidence/post-commit-redelivery.json create mode 100644 evidence/production-containers.json create mode 100644 evidence/production-smoke.txt create mode 100644 evidence/retention-active-recovery.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/staging-retention.txt create mode 100644 evidence/verification.json create mode 100644 infra/Dockerfile.minio create mode 100644 infra/api-policy.json create mode 100644 infra/bootstrap.sh create mode 100644 infra/broker-start.sh create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio-start.sh create mode 100644 infra/operator-policy.json create mode 100644 infra/worker-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_pipeline.sql create mode 100644 requirements.lock create mode 100644 requirements.txt create mode 100644 scripts/integrity.py create mode 100644 scripts/migrate.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/setup.py create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/verify.py create mode 100644 scripts/verify_cleanup.py create mode 100644 scripts/verify_redelivery.py create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/cleanup.py create mode 100644 src/jobs.py create mode 100644 src/operator.py create mode 100644 src/publisher.py create mode 100644 src/storage.py create mode 100644 src/transform.py create mode 100644 src/worker.py create mode 100644 student-tests/dbtool.py create mode 100644 student-tests/maintenance.py create mode 100644 student-tests/pipeline.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..cfb626a8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf +*.cfg text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..84e37e50 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,24 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Брезгунова Анжелика / БВТ2303 +Целевая оценка: 5 +Трек: 2 — доказанная идемпотентность versioned pipeline -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Upload теперь подтверждает durable S3 staging и транзакционные Attachment/Job/Outbox ответом 202 queued. Отдельный worker через persistent Redis Streams публикует original, thumbnail, crop и WebP, атомарно переводит вложение в ready. При остановке worker/API/broker принятая задача восстанавливается из durable intent; дубли не создают дополнительный набор вариантов, старое поколение не перезаписывает новое. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Из собственной lab4 перенесены REST messenger, PostgreSQL sessions/ACL/keyset pagination, две API, message idempotency, private S3, bounded ingress и Compose bootstrap. Добавлены outbox relay, lease/session fencing, retry/backoff/jitter, failed registry, audited replay, version upgrade, staging cleanup и fault-injection campaign. Остальные лабораторные и обязательные контракты/публичные тесты не изменены. + +Запуск: `python scripts/setup.py`, `docker compose --env-file .env up -d --build`, дождаться healthy, затем `python tests/smoke.py --base-url http://127.0.0.1:8085`. Полная методика — [REPORT.md](../REPORT.md). + +Локально 10.10.2026 прошли check, OpenAPI validation, public smoke/persistence/failover, image acceptance и 15 pipeline scenarios. Дополнительно доказаны фактическая redelivery после COMMIT до ack и safe staging retention. Доказательства — `evidence/`. Failover: 72 подтверждения, 2 переходные ошибки, восстановление 2.625 s, без потери/дублей. + +Ограничения: один host и одиночные хранилища; отсутствие HA/backup, worst-case RSS/load study; версия 2 проверяет generation fencing с прежним алгоритмом кодека. Terminal audit/old version results не имеют собственной retention. Доставка at least once, отдельные S3 PUT могут повториться до result COMMIT. Удалённый CI пока не запускался. + +Источники — официальные Redis/Pillow/PostgreSQL/S3/MinIO docs в REPORT.md. База перенесена из собственной lab4; приложение, свои проверки и документация созданы с помощью OpenAI Codex. + +- [x] Обязательный минимум сохранён с переходом upload 201 → 202. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Эквиваленты make check / validate / test выполнены; сохранён реальный вывод. +- [x] Сценарии отказов и выбранного трека воспроизведены. +- [x] Secrets исключены из Git/build context; evidence не содержит tokens. +- [x] README задания сохранён, REPORT.md описывает запуск и ограничения. diff --git a/.github/workflows/solution.yml b/.github/workflows/solution.yml new file mode 100644 index 00000000..579f5b01 --- /dev/null +++ b/.github/workflows/solution.yml @@ -0,0 +1,28 @@ +name: Lab5 solution +on: [push, pull_request] +permissions: + contents: read +jobs: + integration: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-dev.txt + - run: python scripts/setup.py + - run: chmod +x scripts/restart.sh scripts/stop-one.sh + - run: docker compose --env-file .env build + - run: make check validate PYTHON=python + - run: python scripts/verify.py + - run: python scripts/verify_redelivery.py + - run: python scripts/verify_cleanup.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab5-evidence + path: evidence/ + - if: always() + run: docker compose --env-file .env down diff --git a/DEFENSE.md b/DEFENSE.md new file mode 100644 index 00000000..7ec50677 --- /dev/null +++ b/DEFENSE.md @@ -0,0 +1,20 @@ +# Подготовка к защите lab5 + +1. Выполнить `python scripts/setup.py`, `docker compose --env-file .env up -d --build`, затем `python tests/smoke.py --base-url http://127.0.0.1:8085`. +2. Показать `compose.yaml`: две API, PostgreSQL, private MinIO, durable Redis Streams, publisher, отдельный worker, janitor и LB. +3. Открыть `scripts/verify.py` и `evidence/pipeline-results.json`: worker off → 202 → message с queued Attachment → restart API/broker → ready, прежняя сессия и точные bytes. +4. Показать аварии перед публикацией, после публикации, перед S3, после original PUT, перед COMMIT ready и после COMMIT до ack. Fault действительно завершает процесс через os._exit, а не только выбрасывает исключение. Умерший consumer восстанавливается supervisor; publisher — Docker restart policy. +5. Объяснить повторы и replay: managed attempt увеличивается после захвата в БД; Redis redelivery не создаёт новую job. Retry имеет backoff/jitter, постоянная ошибка сразу failed, временная — до 5 попыток. Явный replay audited и запрещён для ready. +6. Показать эксперимент v1 → v2: старые уведомления приходят вокруг новой работы, объекты текущего поколения и статус не меняются. На каждую версию ровно четыре ключа; checksum проверен по реальным bytes. + +**Что сохранено к моменту 202?** Полный validated original в private staging S3, Attachment, Job и Outbox в одной PostgreSQL-транзакции. Redis ещё может быть выключен. Если COMMIT неопределёнен, API не удаляет staging. + +**Publisher confirm и consumer ack?** Redis XADD с appendfsync=always подтверждает запись уведомления в брокер. XACK удаляет его из pending после устойчивого результата или устойчивой регистрации retry/failed в БД. Outbox защищает намерение, если публикация не состоялась или её ответ потерян. + +**Почему это не exactly-once?** XADD и PostgreSQL COMMIT независимы. После crash между ними уведомление публикуется повторно. Worker может снова PUT в тот же детерминированный ключ после crash до result COMMIT. Доставка и отдельные вызовы имеют повторы; логический опубликованный результат защищён идемпотентностью. + +**Lease и lock?** Lease позволяет найти умершего worker; session advisory lock сериализует все побочные эффекты одного attachment. Пока старый процесс держит lock, recovery не захватывает его работу, даже если lease истёк. Умерший процесс освобождает соединение/lock. Сам lease без fencing был бы недостаточен. + +**Лимиты?** 10 MiB, 40M pixels, 4 upload slots/API, 1000 pending по БД, concurrency=2/prefetch=1, 60 секунд wall time, 768 MiB address-space на transform, 2 GiB cgroup всего worker. Подробности и ограничения — REPORT.md. + +**Чего эксперимент не доказывает?** HA после потери host/disка, сохранность при неверных гарантиях fsync оборудования, скорость под production нагрузкой. DB session lock предполагает работающий PostgreSQL и обычную fail-stop модель; split brain/самовольная смена БД за пределами эксперимента. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ae2b9a6f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.12.11-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +COPY requirements.lock . +RUN pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home app +COPY src src +COPY migrations migrations +COPY scripts scripts +COPY tests tests +COPY student-tests student-tests +COPY contracts contracts +COPY lab.json . +USER app +CMD ["python", "-m", "uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8080", "--no-access-log", "--limit-concurrency", "64", "--timeout-keep-alive", "5"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..4d411bdf 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,171 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Автор: Брезгунова Анжелика, БВТ2303. Целевая оценка: 5. Трек 2 — доказанная идемпотентность versioned pipeline.** + +Работа выполнена в `mtusi2026lab5` на основе собственного сервера lab4. Директории остальных лабораторных не редактировались. Обязательные README, COURSE, OpenAPI, JSON Schema и публичные tests сохранены. Исходный commit skeleton: `276dbda`; изменения пока в рабочем каталоге, commit/PR не опубликованы. Шаблон PR заполнен отдельно. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Linux-контейнерами, Compose v2 и Python 3.9+ для публичных тестов. Проверено на Windows/Python 3.11, runtime контейнеров Python 3.12.11. Первый build требует Docker Hub, GitHub и PyPI. Все команды выполнять **из директории lab5**. Compose project `mtusi-lab5`, сети и три persistent volumes отдельные. Наружу открыт только `127.0.0.1:8085`; соседние стенды продолжают работать. -## Архитектура и решения +```sh +python scripts/setup.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +docker compose --env-file .env ps +# Дождаться healthy у api1/api2, затем: +python tests/smoke.py --base-url http://127.0.0.1:8085 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`setup.py` создаёт случайные локальные secrets и `.env` только в lab5, существующие значения сохраняет. Миграции выполняет одноразовый migrate под advisory lock, SQL идемпотентен. Bootstrap создаёт private bucket и отдельных пользователей API/worker/operator, также допускает повтор. Secrets исключены из Git и Docker build context. При проверках уже собранные MinIO/mc нужных версий были доступны в Docker cache; чистый clone собирает их из исходников, соседние файлы не нужны. + +Версии: Python `3.12.11-slim-bookworm`, PostgreSQL `17.6-alpine3.22`, Redis `7.4.5-alpine3.21`, HAProxy `3.0.11-alpine`, MinIO `RELEASE.2025-04-22T22-12-26Z`, mc `RELEASE.2025-04-16T18-13-26Z`. Python зависимости, включая Redis client 6.2.0 и Pillow 11.3.0, закреплены в requirements.lock. MinIO/mc собираются Dockerfile.minio из тегов исходников. `latest` не используется. + +Проверки без Make на Windows и их эквиваленты: + +```sh +python scripts/check_contract.py # make check PYTHON=python +python tests/smoke.py --base-url http://127.0.0.1:8085 +# make test PYTHON=python BASE_URL=http://127.0.0.1:8085 +python -m venv .venv +# Windows: +.venv/Scripts/python -m pip install --no-cache-dir -r requirements-dev.txt +.venv/Scripts/python scripts/validate_openapi.py +# Linux: заменить .venv/Scripts/python на .venv/bin/python +``` + +Полная fault campaign создаёт только синтетические данные и останавливает **только mtusi-lab5**: + +```sh +# Только на Linux, для публичных hooks: +chmod +x scripts/restart.sh scripts/stop-one.sh +python scripts/verify.py +python scripts/verify_redelivery.py +python scripts/verify_cleanup.py +# Вернуть обычные timeout/lease и выключить fault injection: +docker compose --env-file .env up -d --no-build +``` + +verify.py накладывает compose.test.yaml: timeout 6 s, lease 12 s, retry base 0.2 s, ENABLE_FAULTS=1 только у worker/publisher. Обычный стенд: timeout 60 s, lease 90 s, base 1 s, faults выключены. Retention-тест меняет TTL на 1 s только в собственном тестовом процессе; обычный janitor сохраняет 86400 s. CI solution.yml выполняет сборку/проверки и сохраняет evidence; удалённый CI пока не запускался. + +Остановка с сохранением данных: `docker compose --env-file .env stop` или `docker compose --env-file .env down`. Намеренный сброс **только учебных данных lab5**: `docker compose --env-file .env down --volumes`. Эта команда не выполнялась при проверке сохранности. Тесты создают уникальных пользователей и допускают повтор без очистки volumes. + +## Архитектура и границы устойчивости + +```mermaid +flowchart LR + C[Клиент] --> LB[HAProxy] + LB --> A[API 1 и API 2] + A --> S[Private S3 staging] + A --> D[(PostgreSQL: Attachment + Job + Outbox)] + D --> P[Publisher / lease recovery] + P --> R[(Redis Streams + AOF)] + R --> W[Отдельный worker] + W --> T[Transform subprocess] + W --> F[Private S3 results/version] + W --> D +``` + +Перенесены регистрация, Argon2id, Bearer sessions в PostgreSQL, чаты/ACL, история/keyset pagination, две API, общая идемпотентность сообщений, бинарный ingress и private S3. Обязательное изменение: upload 201/ready → **202/queued**, content до ready отвечает 409. Queued/processing можно сразу связать с Message, failed нельзя. Resize выполняет только worker. API всё ещё принимает все bytes и проверяет MIME/целостность/пиксели до принятия; это занимает время и CPU. + +До 202 validated original записан под `staging/{server_uuid}` в MinIO, затем Attachment, Job и Outbox зафиксированы **одной PostgreSQL-транзакцией**. Ephemeral `/tmp` API — только receive buffer. Crash после staging до DB COMMIT оставляет неподтверждённый orphan; janitor удаляет его после retention. При неопределённом COMMIT API не удаляет S3 объект: это могло бы уничтожить принятую задачу. + +PostgreSQL — источник истины состояния и намерения. Publisher читает due outbox и вызывает XADD. Redis: `appendonly yes`, `appendfsync always`, volume `/data`, `maxmemory-policy noeviction`. Ответ XADD подтверждает приём брокером; outbox отмечается опубликованным после ответа. Crash до/после публикации сохраняет намерение и допускает дубль. Каждые 15 s queued intent может публиковаться повторно; потерянное уведомление не блокирует задачу навсегда. Terminal jobs/outbox/audit сохраняются в БД. + +Envelope из jobs.schema.json хранится как JSON в Redis field `job`: schema_version, type, job_id, attachment_id, pipeline_version, attempt, created_at. Bytes, credentials, object keys/URL/filename в брокер не передаются. Worker читает доверенные storage metadata из БД. Invalid envelope попадает в безопасный реестр Streams `attachments:invalid` с message_id/error_code. + +Worker: XREADGROUP, group `image-workers`, XAUTOCLAIM для зависших pending. Prefetch=1 на процесс; concurrency=2, допустимы 1–8. Захват транзакционен: managed attempt увеличивается, фиксируются processing и lease. Session advisory lock на attachment удерживается от claim до завершения S3/DB эффектов. Publisher восстанавливает истёкший processing только если lock свободен. Живой медленный worker не конкурирует с recovery; умерший процесс освобождает lock/соединение. Supervisor заменяет погибший consumer, Docker заменяет погибший publisher. + +Ключ эффекта: `(attachment_id,pipeline_version,variant)`, S3 key `results/{id}/v{version}/{original|thumbnail|crop|optimized}`. Повтор после частичного PUT перезаписывает тот же набор ключей. Ready фиксируется только после четырёх PUT; variants/checksums и job ready публикуются атомарно. После ready дубль проверяет HEAD/metadata checksum эффектов, не делает PUT и не меняет attempts/updated_at. XACK следует за durable ready или durable retry/failed в БД; retry уже имеет outbox intent. После ack выполняется XDEL; pending не обрезается, consumer group одна. + +Crash после COMMIT до ack отдельно проверен фактической redelivery: duplicate_ignored, те же jobs/объекты, удалённый staging, обработанная очередь (`post-commit-redelivery.json`). Старый дубль не переводит ready в processing. Version v2/stale jobs защищены той же блокировкой и сравнением версии с БД. CLI описан в contracts/extensions.md. Версии 1/2 используют одинаковые параметры кодека: эксперимент проверяет поколения/fencing, а не пользу нового кодека. + +## Изображения, ошибки и лимиты + +Original побайтно исходный, SHA-256/size проверяются до обработки и при чтении staging worker. EXIF orientation применяется к вариантам, alpha компонуется на white. Новый RGB image и очищенный info удаляют EXIF/ICC. + +| Вариант | Правило | Кодек | +| --- | --- | --- | +| thumbnail | Вписать в 256×256 без увеличения, размеры floor, минимум 1 px | JPEG quality=85, optimize=true, subsampling=0 | +| crop | Центрированный квадрат 128×128, увеличение малого исходника разрешено | JPEG с теми же параметрами | +| optimized | Вписать в 1280×1280 без увеличения | WebP quality=80, method=6 | + +Resize — LANCZOS. WebP не обещает уменьшение каждого файла. PNG 320×200 даёт JPEG 256×160; портрет 199×401 — 127×256; transparent 17×9 остаётся 17×9; JPEG с EXIF orientation=6 — thumbnail 160×256. Проверены реальные bytes, формат, размеры, checksum, отсутствие EXIF/ICC и white alpha. + +Ограничения: 10 MiB compressed bytes, 40M decoded pixels до распаковки, один кадр PNG/JPEG, 4 upload slots и 16 JSON slots/API. Приём body ограничен 30 s, tmpfs 64 MiB/API. Queued/processing ограничены MAX_PENDING_JOBS=1000 по БД, admission сериализован между репликами; превышение → 429 без нового принятого задания. + +Transform — отдельный subprocess: RLIMIT_AS=768 MiB адресного пространства, RLIMIT_CPU и общий wall-clock alarm JOB_TIMEOUT_SECONDS=60. Не каждое 40M изображение обязано обработаться: ограничение RAM может честно дать failed/image_resource_limit. Worker container 2 GiB/2 CPU, tmpfs 128 MiB; concurrency ограничена. Реальный wall deadline проверен sleep fault, прерванным alarm, с последующим retry/ready. Худший RSS всех допустимых изображений не измерялся. + +Временные network/DB/S3 ошибки и timeout — до **5 попыток**, backoff `min(30,base*2^(attempt-1))*U(0.5,1.5)`. В эксперименте первые ожидания 0.208/0.331 s при base=0.2; ready на третьей попытке. Permanent invalid_image/checksum_mismatch/staging_missing/image_resource_limit → сразу failed; исчерпание временных попыток → retry_exhausted. Public error_code — безопасная константа. Умершая processing job ждёт lease, затем расходует тот же конечный бюджет. + +Failed registry — jobs и CLI `python -m src.operator failed`. Replay требует failed и staging с ожидаемыми size/SHA, audited, сбрасывает бюджет в том же job_id. Ready replay отклонён; успешный результат не запускается заново. После 24 h failed staging может быть удалён, replay тогда запрещён до восстановления input. Audit/metadata сохраняются. + +Staging удаляется после COMMIT ready. Janitor раз в 60 s постранично проверяет staging: queued/processing никогда не истекают; old failed/orphan удаляются после 86400 s. Upload lock и повторный HEAD защищают одновременно принимаемый файл, attachment lock — worker/replay/upgrade. Ошибка зависимости прерывает проход. Ready results сохраняются и до привязки: клиент вправе сослаться позже. Тест TTL=1 s подтвердил сохранность active input, удаление old failed/orphan, защиту fresh orphan и отказ replay без staging; active затем дошёл до ready. + +Readiness зависит от PostgreSQL/S3. Redis/worker не нужны для durable синхронного приёма: intent в outbox. Broker outage не переносит resize в API. Live проверяет процесс. API может PUT только staging и GET staging/results; worker GET/Delete staging и PUT/Get results, operator управляет этими prefix. API Delete и PUT final запрещены в проверке. Bucket anonymous access запрещён, content идёт через авторизованный API без redirect. + +Соответствие .env.example: SESSION_STORE=postgres, две именованные API вместо динамического API_REPLICAS; CACHE_URL_FILE не нужен. APP_HOST/PORT фиксированы CMD на 0.0.0.0:8080, внешний LAB5_PORT=8085. Idempotency TTL фиксирован контрактом на 24 h. S3 endpoint/region/bucket и bytes/pixels заданы Compose, credential paths *_FILE совпадают с secrets. BROKER_URL_FILE создан setup.py; concurrency, max attempts, timeout, lease, retention и max pending настраиваются .env. Время публичного API — UTC/RFC3339. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +**Локальные проверки PASS, 10.10.2026.** Пути ниже относятся к evidence/. Tokens/пароли не сохраняются; ID/hash синтетических данных допустимы. + +| Критерий | Команда/сценарий | Наблюдение | Evidence | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Структура/схемы | check_contract.py / validate_openapi.py | Оба PASS | check.txt, openapi-validation.txt | +| Минимум lab4 → lab5 | tests/smoke.py | ACL/session/Unicode/pagination/idempotency; 202, ready, exact original/JPEG thumbnail | smoke.txt, final-smoke.txt, production-smoke.txt | +| Durable acceptance, отдельные worker/broker | verify.py worker_off_api_broker_restart | Worker off → 202/409 → Message → restart API+broker → ready; queue 1 до/после | pipeline-results.json | +| Persistent AOF | dbtool queue / restart broker | appendonly=yes, appendfsync=always, невычитанная job сохранена | pipeline-results.json | +| Полный restart | tests/resilience.py persistence | Прежний token, Message/original сохранены; StartedAt изменился у всех 9 ролей | persistence.txt, before-restart.json, after-restart.json | +| Две API/failover | tests/resilience.py failover | 72 подтверждения, 2 переходные ошибки, recovery 2.625 s; потерь/дублей нет | failover.txt | +| Варианты, integrity, limits | student-tests/pipeline.py | 4 image profiles, 12 negative upload/filename checks; 12 message repeats → один ID | acceptance.txt | +| Private/scoped S3 | student-tests/pipeline.py | Anonymous 403, API final PUT и staging Delete запрещены | acceptance.txt | +| Outbox DB→broker | before_publish / after_publish / broker_offline | Crash publisher/outage не потеряли 202; ready без новой job вручную | pipeline-results.json | +| Crash S3→COMMIT→ack | before_effect / after_object / before_result_commit / after_result_commit | Реальные os._exit, lease recovery, те же keys/bytes | pipeline-results.json, post-commit-redelivery.json | +| Retry, permanent/temporary, timeout | transient_backoff_jitter / real_job_timeout / *_failed_replay | Ready attempt 3; actual alarm; permanent failed attempt 1, exhausted attempt 5 | pipeline-results.json | +| Failed registry/replay | *_failed_replay | Failed content/link=409, audited replay→ready, ready replay rejected | pipeline-results.json | +| Staging retention | verify_cleanup.py | Active сохранён→ready; expired failed/orphan удалены; fresh protected; expired replay rejected | staging-retention.txt, retention-active-recovery.json | +| Трек 2: duplicate/reorder/stale version | 12_duplicate_deliveries / pipeline_v2_reordered_stale_jobs | Objects/Job неизменны, stale_ignored; по 4 keys/v1,v2, новое поколение не перезаписано | pipeline-results.json | +| Итог | verify.py и две дополнительные проверки | 15 campaign scenarios и public persistence/failover/smoke PASS | verification.json | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Исследование идемпотентности — трек 2 -## Эксперименты и отказы +**Гипотеза:** durable DB intent, at-least-once notifications, session fencing и versioned deterministic keys сохраняют подтверждённую 202 работу в fail-stop модели и не позволяют stale job перезаписать опубликованный новый результат. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Инварианты:** после 202 существуют input/job/outbox; ready означает 4 доступных эффекта с size/SHA; для `(attachment,version,variant)` один key; публикует только текущая версия; успешный дубль не меняет ready/attempts/objects; работа завершается ready/failed, crash не порождает бесконечного retry. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Методика:** verify.py создаёт синтетические users/attachments и PNG fixture 320×200, предварительно проходит smoke/четыре image profiles. Для каждой crash точки новая job; worker/publisher остановлены до установки durable одноразового fault rule. Процесс действительно завершается os._exit. Lease recovery проверяется audit и attempts=2. После ready проверяются actual bytes всех эффектов, current state и число S3 keys. Отдельно: 12 дублей готовой v1; v2 upgrade и старые уведомления вокруг нового результата (6 old + 6 current), сравнение key/size/etag/LastModified. Broker restart выполняется с worker off, доказывая сохранность именно невычитанного сообщения. Повтор campaign создаёт новые IDs. -## Ограничения и следующие шаги +Один прогон на каждую детерминированную точку в итоговом campaign: это проверка заданных окон, не статистическая оценка вероятности сбоя. Ранние диагностические image tests не входят в цифры. Volumes сохранялись, история росла. После COMMIT отдельно дождались фактической redelivery; быстрый ready сам по себе ack не доказывает. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +| Воздействие | Итог | Время сценария, s | +| --- | --- | ---: | +| Worker off, restart API/broker | Queue 1 до/после, same accepted job ready | 14.062 | +| 12 дублей ready | Один job/набор, никаких новых PUT | 5.531 | +| COMMIT принятия → crash до publish | Outbox после restart → ready | 8.344 | +| Publish → crash до outbox mark | Повтор безопасен → ready | 7.594 | +| Crash до effects | Lease recovery, attempt 2 → ready | 20.125 | +| Crash после original PUT | Те же 4 keys, attempt 2 → ready | 20.156 | +| Crash после PUT до result COMMIT | Attempt 2 → ready | 19.422 | +| Crash после COMMIT до ack | Ready attempt 1; поздняя redelivery no-op отдельно доказана | 7.531 до ready | +| Две временные ошибки | Backoff/jitter, attempt 3 → ready | 8.313 | +| Actual wall timeout | Alarm прерывает job, attempt 2 → ready | 13.500 | +| Permanent error + replay | Failed attempt 1 → explicit replay → ready | 12.312 | +| 5 temporary errors + replay | retry_exhausted → replay → ready | 15.782 | +| v2 + reordered stale v1 | V1 superseded, v2 ready, по 4 keys/version | 19.344 | +| Broker offline, restart API | 202 принят, outbox после recovery → ready | 7.703 | +| Invalid envelope/staging cleanup | Safe registry, ready staging удалён | 6.688 | + +Время включает Docker stop/start, polling и test-runner: **это не latency percentile и не SLA**. Worker crash ждёт lease 12 s; обычный lease 90 s увеличит recovery. Accepted input/конечные bytes не потерялись, новое поколение не изменилось от stale job. Это at-least-once и идемпотентный логический результат: отдельный PUT до незавершённого ready может повториться; XADD и PostgreSQL COMMIT не одна распределённая транзакция. + +Среда: Intel Core i7-12700H, 14 cores/20 logical CPU, RAM 16788811776 bytes. Docker Desktop/Engine 29.8.0, WSL2 x86_64, VM 20 CPU/8129257472 bytes RAM. API по 1 CPU/1 GiB; worker 2 CPU/2 GiB; PostgreSQL 1 CPU/384 MiB; S3 1 CPU/512 MiB; broker 0.5 CPU/256 MiB; LB 0.5 CPU/128 MiB; publisher/janitor по верхнему пределу 1 CPU/1 GiB. Raw: host-cpu.json, host-memory.json, docker-environment.json, final-containers.json. RSS/throughput/p95 перегрузки не измерялись: выбран трек 2. + +## Ограничения и источники + +Один host/PostgreSQL/MinIO/Redis/LB не обеспечивают HA при потере host/disка. Проверены fail-stop worker/API/publisher, restarts с volumes, DB→broker gap, broker outage и controlled errors. Split brain, принудительная смена PostgreSQL при живом worker и нарушение fsync оборудованием за пределами модели. Session lock координирует с текущей БД; для HA failover требуется иной fencing protocol. + +Потерянные навсегда S3 bytes требуют backup, из SHA восстановление невозможно. Ready дубль использует HEAD/доверенную SHA metadata; эксперимент дополнительно GET проверил actual bytes, но постоянного bit-rot audit нет. Active staging не истекает при долгом outage; admission limit ограничивает число работ. Terminal audit/outbox, старые version results и consumer records пока без отдельной retention. Version upgrade разрешён из ready и использует прежний original. Queue lag/load tuning не исследованы. compose.test.yaml всегда накладывается на compose.yaml. + +Первичные источники: [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/), [Redis AOF/fsync](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [XAUTOCLAIM](https://redis.io/docs/latest/commands/xautoclaim/), [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [Pillow image/pixel limits](https://pillow.readthedocs.io/en/stable/reference/Image.html), [PostgreSQL advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS), [S3 PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html), [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-04-22T22-12-26Z), [mc release](https://github.com/minio/mc/releases/tag/RELEASE.2025-04-16T18-13-26Z). MinIO/mc — GNU AGPLv3. + +Использованы FastAPI/Starlette, Uvicorn, psycopg, boto3/botocore, Pillow, argon2-cffi, redis-py, PostgreSQL, Redis, HAProxy, MinIO. Собственная база lab4 перенесена с указанием происхождения; приложение/свои проверки/документация lab5 написаны при помощи **OpenAI Codex**, публичные tests — из skeleton. Памятка для защиты — DEFENSE.md. diff --git a/compose.test.yaml b/compose.test.yaml new file mode 100644 index 00000000..7e9d8fab --- /dev/null +++ b/compose.test.yaml @@ -0,0 +1,16 @@ +services: + publisher: + environment: + ENABLE_FAULTS: "1" + JOB_TIMEOUT_SECONDS: "6" + JOB_LEASE_SECONDS: "12" + worker: + environment: + ENABLE_FAULTS: "1" + JOB_TIMEOUT_SECONDS: "6" + JOB_LEASE_SECONDS: "12" + RETRY_BASE_SECONDS: "0.2" + test-runner: + environment: + JOB_TIMEOUT_SECONDS: "6" + JOB_LEASE_SECONDS: "12" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..7e25f18c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,181 @@ +name: mtusi-lab5 +x-app: &app + image: mtusi-messenger:lab5 + build: . + init: true + restart: unless-stopped + read_only: true + tmpfs: ["/tmp:size=67108864,mode=1777"] + mem_limit: 1g + cpus: 1.0 + environment: &environment + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-1000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: [database_url, s3_access_key_id, s3_secret_access_key, cursor_secret, broker_url] + networks: [private] + depends_on: + migrate: {condition: service_completed_successfully} + bootstrap: {condition: service_completed_successfully} + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready',timeout=5)"] + interval: 5s + timeout: 6s + retries: 15 +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres-data:/var/lib/postgresql/data] + networks: [private] + mem_limit: 384m + cpus: 1.0 + restart: unless-stopped + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + s3: + image: mtusi-minio:RELEASE.2025-04-22T22-12-26Z + build: {context: ., dockerfile: infra/Dockerfile.minio, target: server} + entrypoint: [/bin/sh, /config/minio-start.sh] + secrets: [minio_root_user, minio_root_password] + volumes: [s3-data:/data, ./infra/minio-start.sh:/config/minio-start.sh:ro] + networks: [private] + mem_limit: 512m + cpus: 1.0 + restart: unless-stopped + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"] + interval: 3s + timeout: 3s + retries: 30 + bootstrap: + image: mtusi-mc:RELEASE.2025-04-16T18-13-26Z + build: {context: ., dockerfile: infra/Dockerfile.minio, target: client} + entrypoint: [/bin/sh, /config/bootstrap.sh] + secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key, worker_access_key_id, worker_secret_access_key, operator_access_key_id, operator_secret_access_key] + volumes: [./infra:/config:ro] + networks: [private] + depends_on: {s3: {condition: service_healthy}} + broker: + image: redis:7.4.5-alpine3.21 + user: redis + entrypoint: [/bin/sh, /config/broker-start.sh] + secrets: [broker_password] + volumes: [broker-data:/data, ./infra/broker-start.sh:/config/broker-start.sh:ro] + tmpfs: ["/tmp:size=1048576,mode=1777"] + networks: [private] + mem_limit: 256m + cpus: 0.5 + restart: unless-stopped + healthcheck: + test: [CMD-SHELL, 'REDISCLI_AUTH=$$(cat /run/secrets/broker_password) redis-cli ping | grep -q PONG'] + interval: 3s + timeout: 3s + retries: 20 + migrate: + <<: *app + restart: "no" + command: [python, -m, scripts.migrate] + depends_on: {postgres: {condition: service_healthy}} + healthcheck: {disable: true} + api1: + <<: *app + environment: {<<: *environment, INSTANCE_ID: api1} + api2: + <<: *app + environment: {<<: *environment, INSTANCE_ID: api2} + lb: + image: haproxy:3.0.11-alpine + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + ports: ["127.0.0.1:${LAB5_PORT:-8085}:8080"] + networks: [private, edge] + mem_limit: 128m + cpus: 0.5 + restart: unless-stopped + depends_on: {api1: {condition: service_healthy}, api2: {condition: service_healthy}} + publisher: + <<: *app + command: [python, -m, src.publisher] + healthcheck: {disable: true} + worker: + <<: *app + command: [python, -m, src.worker] + mem_limit: 2g + cpus: 2.0 + tmpfs: ["/tmp:size=134217728,mode=1777"] + healthcheck: {disable: true} + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/worker_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/worker_secret_access_key + secrets: [database_url, worker_access_key_id, worker_secret_access_key, broker_url] + janitor: + <<: *app + command: [python, -m, src.cleanup] + healthcheck: {disable: true} + environment: &operator-environment + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/operator_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/operator_secret_access_key + secrets: &operator-secrets [database_url, operator_access_key_id, operator_secret_access_key, broker_url] + operator: + <<: *app + restart: "no" + profiles: [tools] + command: [python, -m, src.operator, failed] + healthcheck: {disable: true} + environment: *operator-environment + secrets: *operator-secrets + test-runner: + <<: *app + restart: "no" + profiles: [tests] + command: [python, student-tests/pipeline.py, acceptance] + healthcheck: {disable: true} + environment: + <<: *operator-environment + ENABLE_FAULTS: "1" + BASE_URL: http://lb:8080 + secrets: [database_url, operator_access_key_id, operator_secret_access_key, broker_url, s3_access_key_id, s3_secret_access_key] + volumes: [./evidence:/evidence] +networks: + private: {internal: true} + edge: {} +volumes: + postgres-data: + s3-data: + broker-data: +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + minio_root_user: {file: ./secrets/minio_root_user} + minio_root_password: {file: ./secrets/minio_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + worker_access_key_id: {file: ./secrets/worker_access_key_id} + worker_secret_access_key: {file: ./secrets/worker_secret_access_key} + operator_access_key_id: {file: ./secrets/operator_access_key_id} + operator_secret_access_key: {file: ./secrets/operator_secret_access_key} + cursor_secret: {file: ./secrets/cursor_secret} + broker_password: {file: ./secrets/broker_password} + broker_url: {file: ./secrets/broker_url} diff --git a/contracts/extensions.md b/contracts/extensions.md new file mode 100644 index 00000000..4f29f26a --- /dev/null +++ b/contracts/extensions.md @@ -0,0 +1,17 @@ +# Операторские расширения lab5 + +Обязательный HTTP API, OpenAPI и jobs.schema.json сохранены. Расширения — только CLI внутри private network; доступа через балансировщик нет. + +```sh +docker compose --profile tools run --rm --no-deps operator python -m src.operator failed +docker compose --profile tools run --rm --no-deps operator python -m src.operator replay ATTACHMENT_UUID +docker compose --profile tools run --rm --no-deps operator python -m src.operator upgrade ATTACHMENT_UUID +``` + +`failed` показывает последние 100 failed jobs с безопасным error_code, попытками и версиями. Это устойчивый PostgreSQL-реестр failed вместо отдельной DLQ. `replay` требует failed и доступный staging с ожидаемыми size/SHA; сбрасывает бюджет попыток, сохраняет job_id, пишет operator_replay. Для ready/queued/processing возвращает ненулевой exit, не запускает успешную job повторно. Старые уведомления сверяются с актуальным состоянием БД. + +`upgrade` требует ready. Под общим с worker advisory lock копирует неизменный оригинал в staging, увеличивает pipeline_version, помечает старые jobs superseded и атомарно создаёт новую job/outbox. Эффекты имеют ключ `results/{attachment_id}/v{pipeline_version}/{variant}`. Версии 1 и 2 используют тот же алгоритм преобразования; эксперимент исследует fencing поколений, а не изменение качества кодека. Внешний контракт не раскрывает pipeline_version; оператор проверяет её в БД. Пока новое поколение обрабатывается, GET content отвечает 409. + +`job_audit` хранит события accepted/published/processing/retry_scheduled/lease_recovered/ready/failed/operator_replay/pipeline_upgrade/stale_ignored/duplicate_ignored. В сообщениях Streams находится ровно envelope из jobs.schema.json, поле Redis `job` содержит JSON; bytes, object keys, URL и credentials отсутствуют. + +Fault injection читает `fault_rules` только при ENABLE_FAULTS=1 из compose.test.yaml. В обычном Compose выключен. Это внутренний инструмент тестов; HTTP для его управления отсутствует. diff --git a/evidence/acceptance.txt b/evidence/acceptance.txt new file mode 100644 index 00000000..2a7e057d --- /dev/null +++ b/evidence/acceptance.txt @@ -0,0 +1 @@ +{"result": "PASS", "images": ["fixture", "tiny_alpha", "portrait", "exif"], "negative_uploads": 12, "concurrent_repeats": 12, "message_ids": 1, "denied_s3_operations": 2} diff --git a/evidence/after-restart.json b/evidence/after-restart.json new file mode 100644 index 00000000..97c3c054 --- /dev/null +++ b/evidence/after-restart.json @@ -0,0 +1,65 @@ +{ + "api1": { + "id": "60efa3f419bfd74a2f77b96c9ccdea043fe4a34f4fb771066ed0d68908f6ec1c", + "started_at": "2026-10-10T18:48:02.9628944Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "api2": { + "id": "4916d370dab864ac1f3c8fef3af7388eaced1ad9e59dfb7f9a45eb7cb30d29dd", + "started_at": "2026-10-10T18:48:02.90235717Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "broker": { + "id": "60e5fffbf85d87f2bccc121186222b5870b271e031a80592a985fe5be3561243", + "started_at": "2026-10-10T18:48:02.712816279Z", + "image": "redis:7.4.5-alpine3.21", + "memory_limit": 268435456, + "cpu_limit": 500000000 + }, + "janitor": { + "id": "c540012dfb508e26137881d6b4ef0e8d162d93aa753f879b9cd6e8554b77a095", + "started_at": "2026-10-10T18:48:02.649538204Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "lb": { + "id": "8f0f73c4ac37cc26c36aec49b228f1f143bbaa68e5ba97ccfe9737b38dc9e325", + "started_at": "2026-10-10T18:48:03.181041346Z", + "image": "haproxy:3.0.11-alpine", + "memory_limit": 134217728, + "cpu_limit": 500000000 + }, + "postgres": { + "id": "bbe0711ca3dd7531626aebc36f6f76e3356b462d311b395249a01d162e1cd4bc", + "started_at": "2026-10-10T18:48:02.819515794Z", + "image": "postgres:17.6-alpine3.22", + "memory_limit": 402653184, + "cpu_limit": 1000000000 + }, + "publisher": { + "id": "fa22076f49afcb5f7167dcf80e8e95c22c38eb5474f266f17f0a5e6159c50eb2", + "started_at": "2026-10-10T18:48:02.197399624Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "s3": { + "id": "481e131cee68518b2e3b5c9f06738d2aab54a80dbf89c93d0a32da328ab0d6bc", + "started_at": "2026-10-10T18:48:02.312938674Z", + "image": "mtusi-minio:RELEASE.2025-04-22T22-12-26Z", + "memory_limit": 536870912, + "cpu_limit": 1000000000 + }, + "worker": { + "id": "2380466d5e891c9cdeb42d4affd52504076ac9fc2510c33927a9085f72e0b303", + "started_at": "2026-10-10T18:48:02.443290696Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 2147483648, + "cpu_limit": 2000000000 + } +} \ No newline at end of file diff --git a/evidence/before-restart.json b/evidence/before-restart.json new file mode 100644 index 00000000..64297635 --- /dev/null +++ b/evidence/before-restart.json @@ -0,0 +1,65 @@ +{ + "api1": { + "id": "60efa3f419bfd74a2f77b96c9ccdea043fe4a34f4fb771066ed0d68908f6ec1c", + "started_at": "2026-10-10T18:47:44.515499761Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "api2": { + "id": "4916d370dab864ac1f3c8fef3af7388eaced1ad9e59dfb7f9a45eb7cb30d29dd", + "started_at": "2026-10-10T18:47:44.426162896Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "broker": { + "id": "60e5fffbf85d87f2bccc121186222b5870b271e031a80592a985fe5be3561243", + "started_at": "2026-10-10T18:47:48.095671327Z", + "image": "redis:7.4.5-alpine3.21", + "memory_limit": 268435456, + "cpu_limit": 500000000 + }, + "janitor": { + "id": "c540012dfb508e26137881d6b4ef0e8d162d93aa753f879b9cd6e8554b77a095", + "started_at": "2026-10-10T18:44:30.392935244Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "lb": { + "id": "8f0f73c4ac37cc26c36aec49b228f1f143bbaa68e5ba97ccfe9737b38dc9e325", + "started_at": "2026-10-10T18:39:18.19823672Z", + "image": "haproxy:3.0.11-alpine", + "memory_limit": 134217728, + "cpu_limit": 500000000 + }, + "postgres": { + "id": "bbe0711ca3dd7531626aebc36f6f76e3356b462d311b395249a01d162e1cd4bc", + "started_at": "2026-10-10T18:39:05.370145141Z", + "image": "postgres:17.6-alpine3.22", + "memory_limit": 402653184, + "cpu_limit": 1000000000 + }, + "publisher": { + "id": "fa22076f49afcb5f7167dcf80e8e95c22c38eb5474f266f17f0a5e6159c50eb2", + "started_at": "2026-10-10T18:46:30.791311925Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "s3": { + "id": "481e131cee68518b2e3b5c9f06738d2aab54a80dbf89c93d0a32da328ab0d6bc", + "started_at": "2026-10-10T18:39:05.203195838Z", + "image": "mtusi-minio:RELEASE.2025-04-22T22-12-26Z", + "memory_limit": 536870912, + "cpu_limit": 1000000000 + }, + "worker": { + "id": "2380466d5e891c9cdeb42d4affd52504076ac9fc2510c33927a9085f72e0b303", + "started_at": "2026-10-10T18:47:32.697034265Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 2147483648, + "cpu_limit": 2000000000 + } +} \ No newline at end of file diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/docker-environment.json b/evidence/docker-environment.json new file mode 100644 index 00000000..ab94ea5c --- /dev/null +++ b/evidence/docker-environment.json @@ -0,0 +1,7 @@ +{ + "ServerVersion": "29.8.0", + "OperatingSystem": "Docker Desktop", + "Architecture": "x86_64", + "NCPU": 20, + "MemTotal": 8129257472 +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..b61b80c2 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 72, "failed_attempts": 2, "first_recovery_seconds": 2.625} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-containers.json b/evidence/final-containers.json new file mode 100644 index 00000000..4a7d7121 --- /dev/null +++ b/evidence/final-containers.json @@ -0,0 +1,65 @@ +{ + "api1": { + "id": "60efa3f419bfd74a2f77b96c9ccdea043fe4a34f4fb771066ed0d68908f6ec1c", + "started_at": "2026-10-10T18:48:02.9628944Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "api2": { + "id": "4916d370dab864ac1f3c8fef3af7388eaced1ad9e59dfb7f9a45eb7cb30d29dd", + "started_at": "2026-10-10T18:48:26.13794421Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "broker": { + "id": "60e5fffbf85d87f2bccc121186222b5870b271e031a80592a985fe5be3561243", + "started_at": "2026-10-10T18:48:02.712816279Z", + "image": "redis:7.4.5-alpine3.21", + "memory_limit": 268435456, + "cpu_limit": 500000000 + }, + "janitor": { + "id": "c540012dfb508e26137881d6b4ef0e8d162d93aa753f879b9cd6e8554b77a095", + "started_at": "2026-10-10T18:48:02.649538204Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "lb": { + "id": "8f0f73c4ac37cc26c36aec49b228f1f143bbaa68e5ba97ccfe9737b38dc9e325", + "started_at": "2026-10-10T18:48:03.181041346Z", + "image": "haproxy:3.0.11-alpine", + "memory_limit": 134217728, + "cpu_limit": 500000000 + }, + "postgres": { + "id": "bbe0711ca3dd7531626aebc36f6f76e3356b462d311b395249a01d162e1cd4bc", + "started_at": "2026-10-10T18:48:02.819515794Z", + "image": "postgres:17.6-alpine3.22", + "memory_limit": 402653184, + "cpu_limit": 1000000000 + }, + "publisher": { + "id": "fa22076f49afcb5f7167dcf80e8e95c22c38eb5474f266f17f0a5e6159c50eb2", + "started_at": "2026-10-10T18:48:02.197399624Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "s3": { + "id": "481e131cee68518b2e3b5c9f06738d2aab54a80dbf89c93d0a32da328ab0d6bc", + "started_at": "2026-10-10T18:48:02.312938674Z", + "image": "mtusi-minio:RELEASE.2025-04-22T22-12-26Z", + "memory_limit": 536870912, + "cpu_limit": 1000000000 + }, + "worker": { + "id": "2380466d5e891c9cdeb42d4affd52504076ac9fc2510c33927a9085f72e0b303", + "started_at": "2026-10-10T18:48:02.443290696Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 2147483648, + "cpu_limit": 2000000000 + } +} \ No newline at end of file diff --git a/evidence/final-smoke.txt b/evidence/final-smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/final-smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/host-cpu.json b/evidence/host-cpu.json new file mode 100644 index 00000000..15a3eabf --- /dev/null +++ b/evidence/host-cpu.json @@ -0,0 +1,5 @@ +{ + "Name": "12th Gen Intel(R) Core(TM) i7-12700H", + "NumberOfCores": 14, + "NumberOfLogicalProcessors": 20 +} diff --git a/evidence/host-memory.json b/evidence/host-memory.json new file mode 100644 index 00000000..f55ed906 --- /dev/null +++ b/evidence/host-memory.json @@ -0,0 +1,3 @@ +{ + "TotalPhysicalMemory": 16788811776 +} diff --git a/evidence/integrity.json b/evidence/integrity.json new file mode 100644 index 00000000..73166ded --- /dev/null +++ b/evidence/integrity.json @@ -0,0 +1,8 @@ +{ + "result": "PASS", + "unchanged_mandatory_files": 17, + "scanned_files": 73, + "private_value_leaks": 0, + "local_secrets_ignored": true, + "scope": "mtusi2026lab5 only" +} \ No newline at end of file diff --git a/evidence/openapi-validation.txt b/evidence/openapi-validation.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/openapi-validation.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/pipeline-results.json b/evidence/pipeline-results.json new file mode 100644 index 00000000..c03c95aa --- /dev/null +++ b/evidence/pipeline-results.json @@ -0,0 +1,2363 @@ +{ + "date": "2026-10-10T18:47:57.598857+00:00", + "scenarios": [ + { + "scenario": "worker_off_api_broker_restart", + "result": "PASS", + "duration_seconds": 14.062, + "queue_before": { + "length": 1, + "aof": { + "appendonly": "yes" + }, + "fsync": { + "appendfsync": "always" + }, + "invalid": 0 + }, + "queue_after": { + "length": 1, + "aof": { + "appendonly": "yes" + }, + "fsync": { + "appendfsync": "always" + }, + "invalid": 0 + }, + "before": { + "api1": { + "id": "60efa3f419bfd74a2f77b96c9ccdea043fe4a34f4fb771066ed0d68908f6ec1c", + "started_at": "2026-10-10T18:44:30.895957951Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "api2": { + "id": "4916d370dab864ac1f3c8fef3af7388eaced1ad9e59dfb7f9a45eb7cb30d29dd", + "started_at": "2026-10-10T18:44:30.731025792Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "broker": { + "id": "60e5fffbf85d87f2bccc121186222b5870b271e031a80592a985fe5be3561243", + "started_at": "2026-10-10T18:39:05.044798019Z", + "image": "redis:7.4.5-alpine3.21", + "memory_limit": 268435456, + "cpu_limit": 500000000 + }, + "janitor": { + "id": "c540012dfb508e26137881d6b4ef0e8d162d93aa753f879b9cd6e8554b77a095", + "started_at": "2026-10-10T18:44:30.392935244Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "lb": { + "id": "8f0f73c4ac37cc26c36aec49b228f1f143bbaa68e5ba97ccfe9737b38dc9e325", + "started_at": "2026-10-10T18:39:18.19823672Z", + "image": "haproxy:3.0.11-alpine", + "memory_limit": 134217728, + "cpu_limit": 500000000 + }, + "postgres": { + "id": "bbe0711ca3dd7531626aebc36f6f76e3356b462d311b395249a01d162e1cd4bc", + "started_at": "2026-10-10T18:39:05.370145141Z", + "image": "postgres:17.6-alpine3.22", + "memory_limit": 402653184, + "cpu_limit": 1000000000 + }, + "publisher": { + "id": "fa22076f49afcb5f7167dcf80e8e95c22c38eb5474f266f17f0a5e6159c50eb2", + "started_at": "2026-10-10T18:44:30.561686671Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "s3": { + "id": "481e131cee68518b2e3b5c9f06738d2aab54a80dbf89c93d0a32da328ab0d6bc", + "started_at": "2026-10-10T18:39:05.203195838Z", + "image": "mtusi-minio:RELEASE.2025-04-22T22-12-26Z", + "memory_limit": 536870912, + "cpu_limit": 1000000000 + } + }, + "after": { + "api1": { + "id": "60efa3f419bfd74a2f77b96c9ccdea043fe4a34f4fb771066ed0d68908f6ec1c", + "started_at": "2026-10-10T18:44:54.559012673Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "api2": { + "id": "4916d370dab864ac1f3c8fef3af7388eaced1ad9e59dfb7f9a45eb7cb30d29dd", + "started_at": "2026-10-10T18:44:54.506035008Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "broker": { + "id": "60e5fffbf85d87f2bccc121186222b5870b271e031a80592a985fe5be3561243", + "started_at": "2026-10-10T18:44:54.42838563Z", + "image": "redis:7.4.5-alpine3.21", + "memory_limit": 268435456, + "cpu_limit": 500000000 + }, + "janitor": { + "id": "c540012dfb508e26137881d6b4ef0e8d162d93aa753f879b9cd6e8554b77a095", + "started_at": "2026-10-10T18:44:30.392935244Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "lb": { + "id": "8f0f73c4ac37cc26c36aec49b228f1f143bbaa68e5ba97ccfe9737b38dc9e325", + "started_at": "2026-10-10T18:39:18.19823672Z", + "image": "haproxy:3.0.11-alpine", + "memory_limit": 134217728, + "cpu_limit": 500000000 + }, + "postgres": { + "id": "bbe0711ca3dd7531626aebc36f6f76e3356b462d311b395249a01d162e1cd4bc", + "started_at": "2026-10-10T18:39:05.370145141Z", + "image": "postgres:17.6-alpine3.22", + "memory_limit": 402653184, + "cpu_limit": 1000000000 + }, + "publisher": { + "id": "fa22076f49afcb5f7167dcf80e8e95c22c38eb5474f266f17f0a5e6159c50eb2", + "started_at": "2026-10-10T18:44:30.561686671Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "s3": { + "id": "481e131cee68518b2e3b5c9f06738d2aab54a80dbf89c93d0a32da328ab0d6bc", + "started_at": "2026-10-10T18:39:05.203195838Z", + "image": "mtusi-minio:RELEASE.2025-04-22T22-12-26Z", + "memory_limit": 536870912, + "cpu_limit": 1000000000 + } + }, + "job": { + "attachment_id": "2311776c-f185-4ad5-969b-acfb725076fd", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "b8b180c5-3014-469a-acce-94dc7bfbb8a4", + "attachment_id": "2311776c-f185-4ad5-969b-acfb725076fd", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:44:51.801435+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:44:51.801435+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:44:51.802827+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:44:51.934452+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:45:02.793778+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:45:03.154110+00:00" + } + ], + "objects": [ + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:45:03.142000+00:00" + }, + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:45:03.148000+00:00" + }, + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:45:03.128000+00:00" + }, + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:45:03.136000+00:00" + } + ] + } + }, + { + "scenario": "12_duplicate_deliveries", + "result": "PASS", + "duration_seconds": 5.531, + "job": { + "attachment_id": "2311776c-f185-4ad5-969b-acfb725076fd", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "b8b180c5-3014-469a-acce-94dc7bfbb8a4", + "attachment_id": "2311776c-f185-4ad5-969b-acfb725076fd", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:44:51.801435+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:44:51.801435+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:44:51.802827+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:44:51.934452+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:45:02.793778+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:45:03.154110+00:00" + }, + { + "event": "duplicate_ignored", + "detail": "", + "created_at": "2026-10-10 18:45:06.532854+00:00" + }, + { + "event": "duplicate_ignored", + "detail": "", + "created_at": "2026-10-10 18:45:06.573765+00:00" + }, + { + "event": "duplicate_ignored", + "detail": "", + "created_at": "2026-10-10 18:45:06.610548+00:00" + } + ], + "objects": [ + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:45:03.142000+00:00" + }, + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:45:03.148000+00:00" + }, + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:45:03.128000+00:00" + }, + { + "key": "results/2311776c-f185-4ad5-969b-acfb725076fd/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:45:03.136000+00:00" + } + ] + } + }, + { + "scenario": "before_publish", + "result": "PASS", + "duration_seconds": 8.344, + "job": { + "attachment_id": "40553411-8a16-4c05-9590-0e0bfe2e9c25", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "22e87082-20df-4cc3-92f2-982775baeaa5", + "attachment_id": "40553411-8a16-4c05-9590-0e0bfe2e9c25", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:45:11.543774+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:45:11.543774+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:45:11.545263+00:00" + }, + { + "event": "fault", + "detail": "before_publish", + "created_at": "2026-10-10 18:45:15.820781+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:45:16.744792+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:45:16.760972+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:45:17.153895+00:00" + } + ], + "objects": [ + { + "key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:45:17.141000+00:00" + }, + { + "key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:45:17.148000+00:00" + }, + { + "key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:45:17.127000+00:00" + }, + { + "key": "results/40553411-8a16-4c05-9590-0e0bfe2e9c25/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:45:17.135000+00:00" + } + ] + } + }, + { + "scenario": "after_publish", + "result": "PASS", + "duration_seconds": 7.594, + "job": { + "attachment_id": "c5e90c2d-77c0-41ff-bf8b-f33a15144a46", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "76f82dc0-43ec-4f47-8590-6adeca600f39", + "attachment_id": "c5e90c2d-77c0-41ff-bf8b-f33a15144a46", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:45:19.833056+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:45:19.833056+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:45:19.834514+00:00" + }, + { + "event": "fault", + "detail": "after_publish", + "created_at": "2026-10-10 18:45:24.128187+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:45:24.287357+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:45:24.685492+00:00" + } + ], + "objects": [ + { + "key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:45:24.673000+00:00" + }, + { + "key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:45:24.679000+00:00" + }, + { + "key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:45:24.656000+00:00" + }, + { + "key": "results/c5e90c2d-77c0-41ff-bf8b-f33a15144a46/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:45:24.667000+00:00" + } + ] + } + }, + { + "scenario": "before_effect", + "result": "PASS", + "duration_seconds": 20.125, + "job": { + "attachment_id": "b0d03f83-691c-4fb3-bbb6-dc9007f099aa", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "50682702-fbc1-4459-8f1e-eb3b0477658d", + "attachment_id": "b0d03f83-691c-4fb3-bbb6-dc9007f099aa", + "pipeline_version": 1, + "attempts": 2, + "status": "ready", + "next_attempt_at": "2026-10-10 18:45:44.322736+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:45:27.389824+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:45:27.391199+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:45:31.775992+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:45:31.792628+00:00" + }, + { + "event": "fault", + "detail": "before_effect", + "created_at": "2026-10-10 18:45:31.900202+00:00" + }, + { + "event": "lease_recovered", + "detail": "", + "created_at": "2026-10-10 18:45:44.326054+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:45:44.362709+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:45:44.381170+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:45:44.759443+00:00" + } + ], + "objects": [ + { + "key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:45:44.746000+00:00" + }, + { + "key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:45:44.752000+00:00" + }, + { + "key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:45:44.727000+00:00" + }, + { + "key": "results/b0d03f83-691c-4fb3-bbb6-dc9007f099aa/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:45:44.737000+00:00" + } + ] + } + }, + { + "scenario": "after_object", + "result": "PASS", + "duration_seconds": 20.156, + "job": { + "attachment_id": "870eaa19-df60-47e6-bd2e-4b21a32c6932", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "75761fff-9bf8-4334-a96c-cbb9716a5a45", + "attachment_id": "870eaa19-df60-47e6-bd2e-4b21a32c6932", + "pipeline_version": 1, + "attempts": 2, + "status": "ready", + "next_attempt_at": "2026-10-10 18:46:04.472182+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:45:47.539339+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:45:47.540888+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:45:51.931669+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:45:52.067678+00:00" + }, + { + "event": "fault", + "detail": "after_object", + "created_at": "2026-10-10 18:45:52.415966+00:00" + }, + { + "event": "lease_recovered", + "detail": "", + "created_at": "2026-10-10 18:46:04.474160+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:46:04.501986+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:46:04.518426+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:46:04.888538+00:00" + } + ], + "objects": [ + { + "key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:46:04.877000+00:00" + }, + { + "key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:46:04.883000+00:00" + }, + { + "key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:46:04.855000+00:00" + }, + { + "key": "results/870eaa19-df60-47e6-bd2e-4b21a32c6932/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:46:04.871000+00:00" + } + ] + } + }, + { + "scenario": "before_result_commit", + "result": "PASS", + "duration_seconds": 19.422, + "job": { + "attachment_id": "6e5144e7-694e-4621-8118-4a1cb898d4e1", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "cf9db327-9dd9-41f6-862b-2bf4e9c4855a", + "attachment_id": "6e5144e7-694e-4621-8118-4a1cb898d4e1", + "pipeline_version": 1, + "attempts": 2, + "status": "ready", + "next_attempt_at": "2026-10-10 18:46:24.124389+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:46:07.706046+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:07.707357+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:12.081821+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:46:12.098328+00:00" + }, + { + "event": "fault", + "detail": "before_result_commit", + "created_at": "2026-10-10 18:46:12.463556+00:00" + }, + { + "event": "lease_recovered", + "detail": "", + "created_at": "2026-10-10 18:46:24.126376+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:46:24.161674+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:46:24.179411+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:46:24.551037+00:00" + } + ], + "objects": [ + { + "key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:46:24.533000+00:00" + }, + { + "key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:46:24.541000+00:00" + }, + { + "key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:46:24.514000+00:00" + }, + { + "key": "results/6e5144e7-694e-4621-8118-4a1cb898d4e1/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:46:24.524000+00:00" + } + ] + } + }, + { + "scenario": "after_result_commit", + "result": "PASS", + "duration_seconds": 7.531, + "job": { + "attachment_id": "eb0730e0-8e8d-4ddb-976f-fba32e775fa7", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "1ca4e4b8-3a7c-408a-ace8-2edef424137f", + "attachment_id": "eb0730e0-8e8d-4ddb-976f-fba32e775fa7", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:46:27.122611+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:46:27.122611+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:27.123942+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:31.492190+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:46:31.512682+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:46:31.890810+00:00" + }, + { + "event": "fault", + "detail": "after_result_commit", + "created_at": "2026-10-10 18:46:31.893146+00:00" + } + ], + "objects": [ + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:46:31.878000+00:00" + }, + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:46:31.883000+00:00" + }, + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:46:31.863000+00:00" + }, + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:46:31.872000+00:00" + } + ] + } + }, + { + "scenario": "transient_backoff_jitter", + "result": "PASS", + "duration_seconds": 8.313, + "job": { + "attachment_id": "41bdba4c-a10e-4050-acb3-60925a86c954", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "34f9c7ea-64d3-401c-bddd-3b7804afc912", + "attachment_id": "41bdba4c-a10e-4050-acb3-60925a86c954", + "pipeline_version": 1, + "attempts": 3, + "status": "ready", + "next_attempt_at": "2026-10-10 18:46:39.870756+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:46:34.546800+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:34.548217+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:34.672455+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:46:38.802511+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:46:38.915779+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.208", + "created_at": "2026-10-10 18:46:38.919001+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:46:39.425728+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:46:39.442217+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:46:39.537163+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.331", + "created_at": "2026-10-10 18:46:39.541005+00:00" + }, + { + "event": "published", + "detail": "3", + "created_at": "2026-10-10 18:46:39.985737+00:00" + }, + { + "event": "processing", + "detail": "3", + "created_at": "2026-10-10 18:46:39.996651+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:46:40.271177+00:00" + } + ], + "objects": [ + { + "key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:46:40.259000+00:00" + }, + { + "key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:46:40.265000+00:00" + }, + { + "key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:46:40.242000+00:00" + }, + { + "key": "results/41bdba4c-a10e-4050-acb3-60925a86c954/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:46:40.254000+00:00" + } + ] + }, + "delays": [ + 0.208, + 0.331 + ] + }, + { + "scenario": "real_job_timeout", + "result": "PASS", + "duration_seconds": 13.5, + "job": { + "attachment_id": "f39e4834-f876-4ebb-991a-e3f65a00cb3f", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "5316353d-2c88-4fdc-a43b-c59defa4f39e", + "attachment_id": "f39e4834-f876-4ebb-991a-e3f65a00cb3f", + "pipeline_version": 1, + "attempts": 2, + "status": "ready", + "next_attempt_at": "2026-10-10 18:46:53.447364+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:46:42.857802+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:42.859391+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:43.156320+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:46:47.167743+00:00" + }, + { + "event": "fault", + "detail": "hang", + "created_at": "2026-10-10 18:46:47.280945+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.168", + "created_at": "2026-10-10 18:46:53.280629+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:46:53.635345+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:46:53.648406+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:46:53.921261+00:00" + } + ], + "objects": [ + { + "key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:46:53.910000+00:00" + }, + { + "key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:46:53.915000+00:00" + }, + { + "key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:46:53.892000+00:00" + }, + { + "key": "results/f39e4834-f876-4ebb-991a-e3f65a00cb3f/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:46:53.904000+00:00" + } + ] + } + }, + { + "scenario": "permanent_failed_replay", + "result": "PASS", + "duration_seconds": 12.312, + "failed": { + "attachment_id": "c5c5c3e0-e1da-44ef-94bc-053c953fe89f", + "status": "failed", + "version": 1, + "error_code": "invalid_image", + "variants": [], + "jobs": [ + { + "id": "0ccb0c61-184e-48f8-9550-1416f6c42d43", + "attachment_id": "c5c5c3e0-e1da-44ef-94bc-053c953fe89f", + "pipeline_version": 1, + "attempts": 1, + "status": "failed", + "next_attempt_at": "2026-10-10 18:46:56.361949+00:00", + "lease_until": null, + "error_code": "invalid_image", + "created_at": "2026-10-10 18:46:56.361949+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:56.363672+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:56.813488+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:00.695306+00:00" + }, + { + "event": "fault", + "detail": "permanent", + "created_at": "2026-10-10 18:47:00.802231+00:00" + }, + { + "event": "failed", + "detail": "invalid_image", + "created_at": "2026-10-10 18:47:00.805362+00:00" + } + ], + "objects": [] + }, + "recovered": { + "attachment_id": "c5c5c3e0-e1da-44ef-94bc-053c953fe89f", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "0ccb0c61-184e-48f8-9550-1416f6c42d43", + "attachment_id": "c5c5c3e0-e1da-44ef-94bc-053c953fe89f", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:47:03.924651+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:46:56.361949+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:56.363672+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:56.813488+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:00.695306+00:00" + }, + { + "event": "fault", + "detail": "permanent", + "created_at": "2026-10-10 18:47:00.802231+00:00" + }, + { + "event": "failed", + "detail": "invalid_image", + "created_at": "2026-10-10 18:47:00.805362+00:00" + }, + { + "event": "operator_replay", + "detail": "", + "created_at": "2026-10-10 18:47:03.926220+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:04.198349+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:04.217933+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:04.578636+00:00" + } + ], + "objects": [ + { + "key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:04.566000+00:00" + }, + { + "key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:04.572000+00:00" + }, + { + "key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:04.549000+00:00" + }, + { + "key": "results/c5c5c3e0-e1da-44ef-94bc-053c953fe89f/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:04.560000+00:00" + } + ] + } + }, + { + "scenario": "transient_failed_replay", + "result": "PASS", + "duration_seconds": 15.782, + "failed": { + "attachment_id": "6e64790a-a0b1-44ad-b540-71792e81f13c", + "status": "failed", + "version": 1, + "error_code": "retry_exhausted", + "variants": [], + "jobs": [ + { + "id": "fcf21e14-5451-4259-a92a-b803f2e0a87a", + "attachment_id": "6e64790a-a0b1-44ad-b540-71792e81f13c", + "pipeline_version": 1, + "attempts": 5, + "status": "failed", + "next_attempt_at": "2026-10-10 18:47:16.025182+00:00", + "lease_until": null, + "error_code": "retry_exhausted", + "created_at": "2026-10-10 18:47:08.734888+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:08.736135+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:08.938782+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:12.969818+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:13.069508+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.118", + "created_at": "2026-10-10 18:47:13.073277+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:47:13.688331+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:47:13.706964+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:13.810060+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.373", + "created_at": "2026-10-10 18:47:13.813623+00:00" + }, + { + "event": "published", + "detail": "3", + "created_at": "2026-10-10 18:47:14.247477+00:00" + }, + { + "event": "processing", + "detail": "3", + "created_at": "2026-10-10 18:47:14.260010+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:14.271238+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.448", + "created_at": "2026-10-10 18:47:14.274993+00:00" + }, + { + "event": "published", + "detail": "4", + "created_at": "2026-10-10 18:47:14.803451+00:00" + }, + { + "event": "processing", + "detail": "4", + "created_at": "2026-10-10 18:47:14.815004+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:14.827788+00:00" + }, + { + "event": "retry_scheduled", + "detail": "1.195", + "created_at": "2026-10-10 18:47:14.830446+00:00" + }, + { + "event": "published", + "detail": "5", + "created_at": "2026-10-10 18:47:16.404441+00:00" + }, + { + "event": "processing", + "detail": "5", + "created_at": "2026-10-10 18:47:16.420673+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:16.432742+00:00" + }, + { + "event": "failed", + "detail": "retry_exhausted", + "created_at": "2026-10-10 18:47:16.435456+00:00" + } + ], + "objects": [] + }, + "recovered": { + "attachment_id": "6e64790a-a0b1-44ad-b540-71792e81f13c", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "fcf21e14-5451-4259-a92a-b803f2e0a87a", + "attachment_id": "6e64790a-a0b1-44ad-b540-71792e81f13c", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:47:19.638439+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:08.734888+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:08.736135+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:08.938782+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:12.969818+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:13.069508+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.118", + "created_at": "2026-10-10 18:47:13.073277+00:00" + }, + { + "event": "published", + "detail": "2", + "created_at": "2026-10-10 18:47:13.688331+00:00" + }, + { + "event": "processing", + "detail": "2", + "created_at": "2026-10-10 18:47:13.706964+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:13.810060+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.373", + "created_at": "2026-10-10 18:47:13.813623+00:00" + }, + { + "event": "published", + "detail": "3", + "created_at": "2026-10-10 18:47:14.247477+00:00" + }, + { + "event": "processing", + "detail": "3", + "created_at": "2026-10-10 18:47:14.260010+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:14.271238+00:00" + }, + { + "event": "retry_scheduled", + "detail": "0.448", + "created_at": "2026-10-10 18:47:14.274993+00:00" + }, + { + "event": "published", + "detail": "4", + "created_at": "2026-10-10 18:47:14.803451+00:00" + }, + { + "event": "processing", + "detail": "4", + "created_at": "2026-10-10 18:47:14.815004+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:14.827788+00:00" + }, + { + "event": "retry_scheduled", + "detail": "1.195", + "created_at": "2026-10-10 18:47:14.830446+00:00" + }, + { + "event": "published", + "detail": "5", + "created_at": "2026-10-10 18:47:16.404441+00:00" + }, + { + "event": "processing", + "detail": "5", + "created_at": "2026-10-10 18:47:16.420673+00:00" + }, + { + "event": "fault", + "detail": "transient", + "created_at": "2026-10-10 18:47:16.432742+00:00" + }, + { + "event": "failed", + "detail": "retry_exhausted", + "created_at": "2026-10-10 18:47:16.435456+00:00" + }, + { + "event": "operator_replay", + "detail": "", + "created_at": "2026-10-10 18:47:19.639665+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:20.102104+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:20.111662+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:20.383337+00:00" + } + ], + "objects": [ + { + "key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:20.367000+00:00" + }, + { + "key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:20.375000+00:00" + }, + { + "key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:20.353000+00:00" + }, + { + "key": "results/6e64790a-a0b1-44ad-b540-71792e81f13c/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:20.361000+00:00" + } + ] + } + }, + { + "scenario": "pipeline_v2_reordered_stale_jobs", + "result": "PASS", + "duration_seconds": 19.344, + "v1": { + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "9d71a35a-0eab-4286-90f9-7a02aa789296", + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:47:23.911297+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:23.911298+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:23.912898+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:24.332530+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:24.344343+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:24.641792+00:00" + } + ], + "objects": [ + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:24.628000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:24.634000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:24.612000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:24.621000+00:00" + } + ] + }, + "v2": { + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "status": "ready", + "version": 2, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "9d71a35a-0eab-4286-90f9-7a02aa789296", + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "pipeline_version": 1, + "attempts": 1, + "status": "superseded", + "next_attempt_at": "2026-10-10 18:47:23.911297+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:23.911298+00:00" + }, + { + "id": "b7559722-70f9-48fa-9ce3-193cf498218f", + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "pipeline_version": 2, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:47:28.706910+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:28.706910+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:23.912898+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:24.332530+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:24.344343+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:24.641792+00:00" + }, + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:28.707928+00:00" + }, + { + "event": "pipeline_upgrade", + "detail": "", + "created_at": "2026-10-10 18:47:28.708288+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:29.075280+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:33.381877+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:33.735591+00:00" + } + ], + "objects": [ + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:24.628000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:24.634000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:24.612000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:24.621000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:33.724000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:33.730000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:33.706000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:33.718000+00:00" + } + ] + }, + "final": { + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "status": "ready", + "version": 2, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "9d71a35a-0eab-4286-90f9-7a02aa789296", + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "pipeline_version": 1, + "attempts": 1, + "status": "superseded", + "next_attempt_at": "2026-10-10 18:47:23.911297+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:23.911298+00:00" + }, + { + "id": "b7559722-70f9-48fa-9ce3-193cf498218f", + "attachment_id": "0e686443-7c32-4f0a-bed8-df5f96e2bb2f", + "pipeline_version": 2, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:47:28.706910+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:28.706910+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:23.912898+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:24.332530+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:24.344343+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:24.641792+00:00" + }, + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:28.707928+00:00" + }, + { + "event": "pipeline_upgrade", + "detail": "", + "created_at": "2026-10-10 18:47:28.708288+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:29.075280+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:33.381877+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:33.735591+00:00" + }, + { + "event": "stale_ignored", + "detail": "", + "created_at": "2026-10-10 18:47:37.248293+00:00" + }, + { + "event": "stale_ignored", + "detail": "", + "created_at": "2026-10-10 18:47:37.254741+00:00" + }, + { + "event": "stale_ignored", + "detail": "", + "created_at": "2026-10-10 18:47:37.278222+00:00" + }, + { + "event": "duplicate_ignored", + "detail": "", + "created_at": "2026-10-10 18:47:38.876921+00:00" + }, + { + "event": "duplicate_ignored", + "detail": "", + "created_at": "2026-10-10 18:47:39.035646+00:00" + } + ], + "objects": [ + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:24.628000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:24.634000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:24.612000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:24.621000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:33.724000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:33.730000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:33.706000+00:00" + }, + { + "key": "results/0e686443-7c32-4f0a-bed8-df5f96e2bb2f/v2/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:33.718000+00:00" + } + ] + } + }, + { + "scenario": "broker_offline_outbox_recovery", + "result": "PASS", + "duration_seconds": 7.703, + "job": { + "attachment_id": "3702790a-2c0f-4152-86a4-5ea5dff46327", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "de6f3c72-265d-4db2-bd6e-e98dde081f7c", + "attachment_id": "3702790a-2c0f-4152-86a4-5ea5dff46327", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:47:43.879648+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:47:43.879649+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:47:43.881243+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:47:48.598652+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:47:48.607760+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:47:48.883251+00:00" + } + ], + "objects": [ + { + "key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:47:48.871000+00:00" + }, + { + "key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:47:48.877000+00:00" + }, + { + "key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:47:48.855000+00:00" + }, + { + "key": "results/3702790a-2c0f-4152-86a4-5ea5dff46327/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:47:48.864000+00:00" + } + ] + } + }, + { + "scenario": "invalid_envelope_and_staging_cleanup", + "result": "PASS", + "duration_seconds": 6.688, + "queue": { + "length": 0, + "aof": { + "appendonly": "yes" + }, + "fsync": { + "appendfsync": "always" + }, + "invalid": 1 + }, + "staging_count": 0 + } + ] +} \ No newline at end of file diff --git a/evidence/post-commit-redelivery.json b/evidence/post-commit-redelivery.json new file mode 100644 index 00000000..b28dee8d --- /dev/null +++ b/evidence/post-commit-redelivery.json @@ -0,0 +1,119 @@ +{ + "result": "PASS", + "job": { + "attachment_id": "eb0730e0-8e8d-4ddb-976f-fba32e775fa7", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "1ca4e4b8-3a7c-408a-ace8-2edef424137f", + "attachment_id": "eb0730e0-8e8d-4ddb-976f-fba32e775fa7", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:46:27.122611+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:46:27.122611+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:46:27.123942+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:46:31.492190+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:46:31.512682+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:46:31.890810+00:00" + }, + { + "event": "fault", + "detail": "after_result_commit", + "created_at": "2026-10-10 18:46:31.893146+00:00" + }, + { + "event": "duplicate_ignored", + "detail": "", + "created_at": "2026-10-10 18:46:47.305597+00:00" + } + ], + "objects": [ + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:46:31.878000+00:00" + }, + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:46:31.883000+00:00" + }, + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:46:31.863000+00:00" + }, + { + "key": "results/eb0730e0-8e8d-4ddb-976f-fba32e775fa7/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:46:31.872000+00:00" + } + ] + }, + "queue": { + "length": 0, + "aof": { + "appendonly": "yes" + }, + "fsync": { + "appendfsync": "always" + }, + "invalid": 1 + } +} \ No newline at end of file diff --git a/evidence/production-containers.json b/evidence/production-containers.json new file mode 100644 index 00000000..f2550066 --- /dev/null +++ b/evidence/production-containers.json @@ -0,0 +1,65 @@ +{ + "api1": { + "id": "e5506ad77a3e78452a2b6236940e6f1a88dcc09a1130615ed372895058e1007f", + "started_at": "2026-10-10T18:51:33.307922314Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "api2": { + "id": "cdb2985b7a90d988a472e3a95575b40210a668a5e03f3c61e1eee808c9207602", + "started_at": "2026-10-10T18:51:33.826863887Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "broker": { + "id": "60e5fffbf85d87f2bccc121186222b5870b271e031a80592a985fe5be3561243", + "started_at": "2026-10-10T18:48:02.712816279Z", + "image": "redis:7.4.5-alpine3.21", + "memory_limit": 268435456, + "cpu_limit": 500000000 + }, + "janitor": { + "id": "54f793a1d24d2007fb51a0e4fa05d5891b8da36ab4db1e0cc27e47305559fbef", + "started_at": "2026-10-10T18:51:33.659898913Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "lb": { + "id": "8f0f73c4ac37cc26c36aec49b228f1f143bbaa68e5ba97ccfe9737b38dc9e325", + "started_at": "2026-10-10T18:48:03.181041346Z", + "image": "haproxy:3.0.11-alpine", + "memory_limit": 134217728, + "cpu_limit": 500000000 + }, + "postgres": { + "id": "bbe0711ca3dd7531626aebc36f6f76e3356b462d311b395249a01d162e1cd4bc", + "started_at": "2026-10-10T18:48:02.819515794Z", + "image": "postgres:17.6-alpine3.22", + "memory_limit": 402653184, + "cpu_limit": 1000000000 + }, + "publisher": { + "id": "08f43d66ffc93acb166e6b22e086481c5cc68c5ba4c71b07ed7d07d8f852221a", + "started_at": "2026-10-10T18:51:33.133551482Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 1073741824, + "cpu_limit": 1000000000 + }, + "s3": { + "id": "481e131cee68518b2e3b5c9f06738d2aab54a80dbf89c93d0a32da328ab0d6bc", + "started_at": "2026-10-10T18:48:02.312938674Z", + "image": "mtusi-minio:RELEASE.2025-04-22T22-12-26Z", + "memory_limit": 536870912, + "cpu_limit": 1000000000 + }, + "worker": { + "id": "0087f01d5747626eec0ec61e83df1863523448499e940beb70b31339c5103940", + "started_at": "2026-10-10T18:51:33.498882247Z", + "image": "mtusi-messenger:lab5", + "memory_limit": 2147483648, + "cpu_limit": 2000000000 + } +} \ No newline at end of file diff --git a/evidence/production-smoke.txt b/evidence/production-smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/production-smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/retention-active-recovery.json b/evidence/retention-active-recovery.json new file mode 100644 index 00000000..46021a23 --- /dev/null +++ b/evidence/retention-active-recovery.json @@ -0,0 +1,99 @@ +{ + "result": "PASS", + "job": { + "attachment_id": "ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6", + "status": "ready", + "version": 1, + "error_code": null, + "variants": [ + { + "name": "thumbnail", + "width": 256, + "height": 160, + "sha256": "8fac43e7b4e9148871b7ce4373cdf954bdacad9ba8679847f27b62128e866fc6", + "object_key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/thumbnail", + "size_bytes": 6981, + "content_type": "image/jpeg" + }, + { + "name": "crop", + "width": 128, + "height": 128, + "sha256": "7894b8099ad56bc8a8771eab1f52850a5e4bd07062b68a33c8a4954a08312270", + "object_key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/crop", + "size_bytes": 2858, + "content_type": "image/jpeg" + }, + { + "name": "optimized", + "width": 320, + "height": 200, + "sha256": "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24", + "object_key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/optimized", + "size_bytes": 2344, + "content_type": "image/webp" + } + ], + "jobs": [ + { + "id": "0bb672c2-6589-4868-8575-8150cc12a22e", + "attachment_id": "ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6", + "pipeline_version": 1, + "attempts": 1, + "status": "ready", + "next_attempt_at": "2026-10-10 18:51:20.509622+00:00", + "lease_until": null, + "error_code": null, + "created_at": "2026-10-10 18:51:20.509622+00:00" + } + ], + "events": [ + { + "event": "accepted", + "detail": "", + "created_at": "2026-10-10 18:51:20.511121+00:00" + }, + { + "event": "published", + "detail": "1", + "created_at": "2026-10-10 18:51:27.153825+00:00" + }, + { + "event": "processing", + "detail": "1", + "created_at": "2026-10-10 18:51:27.305794+00:00" + }, + { + "event": "ready", + "detail": "", + "created_at": "2026-10-10 18:51:27.690925+00:00" + } + ], + "objects": [ + { + "key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/crop", + "size": 2858, + "etag": "\"38275e817df1512cfc45b119ee7d8af2\"", + "modified": "2026-10-10 18:51:27.679000+00:00" + }, + { + "key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/optimized", + "size": 2344, + "etag": "\"364f200d7d03b39725dbebecac18f2be\"", + "modified": "2026-10-10 18:51:27.685000+00:00" + }, + { + "key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/original", + "size": 111817, + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "modified": "2026-10-10 18:51:27.663000+00:00" + }, + { + "key": "results/ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6/v1/thumbnail", + "size": 6981, + "etag": "\"22d266a80d796be51d89a8690a065614\"", + "modified": "2026-10-10 18:51:27.672000+00:00" + } + ] + } +} \ No newline at end of file diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/staging-retention.txt b/evidence/staging-retention.txt new file mode 100644 index 00000000..865bbcaf --- /dev/null +++ b/evidence/staging-retention.txt @@ -0,0 +1 @@ +{"result": "PASS", "removed": 2, "active_preserved": "ed1fcdc1-eef3-4bb0-ad1f-feb2622a54c6", "expired_failed": "4359a166-3817-43ed-9215-c5dfa9d828c0", "old_orphan_removed": true, "fresh_orphan_preserved": true, "expired_replay_denied": true} diff --git a/evidence/verification.json b/evidence/verification.json new file mode 100644 index 00000000..e1a8633c --- /dev/null +++ b/evidence/verification.json @@ -0,0 +1,14 @@ +{ + "result": "PASS", + "pipeline_scenarios": 15, + "public_smoke": true, + "public_persistence": true, + "public_failover": true, + "image_acceptance": true, + "base_url": "http://127.0.0.1:8085", + "post_commit_redelivery": true, + "staging_retention": true, + "openapi_validation": true, + "mandatory_integrity": true, + "production_smoke": true +} \ No newline at end of file diff --git a/infra/Dockerfile.minio b/infra/Dockerfile.minio new file mode 100644 index 00000000..3800cae0 --- /dev/null +++ b/infra/Dockerfile.minio @@ -0,0 +1,19 @@ +FROM golang:1.24.2-bookworm AS server-build +RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /source +WORKDIR /source +RUN CGO_ENABLED=0 go build -trimpath -ldflags="$(go run buildscripts/gen-ldflags.go)" -o /out/minio . + +FROM golang:1.24.2-bookworm AS client-build +RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /source +WORKDIR /source +RUN CGO_ENABLED=0 go build -trimpath -ldflags="$(go run buildscripts/gen-ldflags.go)" -o /out/mc . + +FROM python:3.12.11-slim-bookworm AS server +COPY --from=server-build /out/minio /usr/local/bin/minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z" +ENTRYPOINT ["minio"] + +FROM python:3.12.11-slim-bookworm AS client +COPY --from=client-build /out/mc /usr/local/bin/mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z" +ENTRYPOINT ["mc"] diff --git a/infra/api-policy.json b/infra/api-policy.json new file mode 100644 index 00000000..ba2fee97 --- /dev/null +++ b/infra/api-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]},{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject"],"Resource":["arn:aws:s3:::messenger-attachments/staging/*"]},{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::messenger-attachments/results/*"]}]} diff --git a/infra/bootstrap.sh b/infra/bootstrap.sh new file mode 100644 index 00000000..49e62117 --- /dev/null +++ b/infra/bootstrap.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mc alias set local http://s3:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +for role in api worker operator; do + mc admin policy create local messenger-$role /config/$role-policy.json >/dev/null + case "$role" in + api) key=s3_access_key_id; secret=s3_secret_access_key ;; + worker) key=worker_access_key_id; secret=worker_secret_access_key ;; + operator) key=operator_access_key_id; secret=operator_secret_access_key ;; + esac + mc admin user add local "$(cat /run/secrets/$key)" "$(cat /run/secrets/$secret)" >/dev/null + mc admin policy attach local messenger-$role --user "$(cat /run/secrets/$key)" >/dev/null +done +echo 'Private bucket and scoped API/worker/operator users ready.' diff --git a/infra/broker-start.sh b/infra/broker-start.sh new file mode 100644 index 00000000..b88ca976 --- /dev/null +++ b/infra/broker-start.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +umask 077 +cat > /tmp/redis.conf < 0); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS variants jsonb NOT NULL DEFAULT '[]'; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code text; +CREATE TABLE IF NOT EXISTS jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK(pipeline_version > 0), + attempts integer NOT NULL DEFAULT 0, status text NOT NULL DEFAULT 'queued' + CHECK(status IN ('queued','processing','ready','failed','superseded')), + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), lease_until timestamptz, + error_code text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX IF NOT EXISTS jobs_due ON jobs(status,next_attempt_at); +CREATE TABLE IF NOT EXISTS outbox ( + job_id uuid PRIMARY KEY REFERENCES jobs(id), + published_at timestamptz, publish_count integer NOT NULL DEFAULT 0 +); +CREATE TABLE IF NOT EXISTS job_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id), + event text NOT NULL, detail text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS fault_rules ( + job_id uuid PRIMARY KEY REFERENCES jobs(id), point text NOT NULL, + remaining integer NOT NULL DEFAULT 1 +); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..2714661c --- /dev/null +++ b/requirements.lock @@ -0,0 +1,27 @@ +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +boto3==1.38.46 +botocore==1.38.46 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +pillow==11.3.0 +psycopg==3.2.9 +psycopg-binary==3.2.9 +pycparser==3.11 +pydantic==2.14.0 +pydantic_core==2.50.0 +python-dateutil==2.9.0.post0 +s3transfer==0.13.1 +six==1.17.0 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 +redis==6.2.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c3f37ae0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +boto3==1.38.46 +Pillow==11.3.0 +argon2-cffi==25.1.0 +redis==6.2.0 diff --git a/scripts/integrity.py b/scripts/integrity.py new file mode 100644 index 00000000..878e4bbe --- /dev/null +++ b/scripts/integrity.py @@ -0,0 +1,30 @@ +"""Read-only Git checks and private-value scan; prints only names/counts.""" +import json +from pathlib import Path +import subprocess +import sys +ROOT = Path(__file__).resolve().parents[1] + + +def git(*args): + return subprocess.check_output(['git',*args],cwd=ROOT,text=True,encoding='utf-8').strip() + + +public = git('ls-files','--','README.md','COURSE.md','CONTRIBUTING.md','tests','contracts').splitlines() +changed = [] +for name in public: + if git('hash-object','--path',name,name)!=git('rev-parse','HEAD:'+name): + changed.append(name) +assert not changed,'Mandatory skeleton changed: '+', '.join(changed) +private = [p.read_bytes().strip() for p in (ROOT/'secrets').iterdir() if p.is_file()] +scan = [] +for pattern in ('*.md','*.json','*.txt','*.yaml','*.yml','*.py'): + scan.extend(p for p in ROOT.rglob(pattern) if not set(p.relative_to(ROOT).parts)&{'.git','.venv','secrets','__pycache__'}) +leaks = [str(p.relative_to(ROOT)) for p in scan if any(value and value in p.read_bytes() for value in private)] +assert not leaks,'Private value found in '+', '.join(leaks) +ignored = subprocess.run(['git','check-ignore','.env','secrets/database_url','secrets/broker_url'],cwd=ROOT,capture_output=True,text=True) +assert ignored.returncode==0 and len(ignored.stdout.splitlines())==3 +result = {'result':'PASS','unchanged_mandatory_files':len(public),'scanned_files':len(set(scan)), + 'private_value_leaks':0,'local_secrets_ignored':True,'scope':'mtusi2026lab5 only'} +(ROOT/'evidence/integrity.json').write_text(json.dumps(result,indent=2),encoding='utf-8') +print(json.dumps(result)) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..43942bcd --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,8 @@ +from pathlib import Path +from src.storage import db + +with db() as conn: + conn.execute('SELECT pg_advisory_xact_lock(440001)') + for path in sorted(Path('migrations').glob('*.sql')): + conn.execute(path.read_text()) +print('Schema ready (idempotent migration).') diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..95b520cc --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +docker compose --env-file .env restart api1 api2 postgres s3 broker publisher worker janitor lb +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..d226d2aa --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart api1 api2 postgres s3 broker publisher worker janitor lb diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 00000000..f919446a --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,25 @@ +"""Generate local random secrets inside lab5 only; preserve existing files.""" +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +folder = root / 'secrets' +folder.mkdir(exist_ok=True) +values = {} +for name in ('postgres_password','minio_root_user','minio_root_password','s3_access_key_id', + 's3_secret_access_key','worker_access_key_id','worker_secret_access_key', + 'operator_access_key_id','operator_secret_access_key','broker_password','cursor_secret'): + target = folder / name + if not target.exists(): + target.write_text(secrets.token_hex(24),encoding='utf-8') + values[name] = target.read_text().strip() +target = folder / 'database_url' +if not target.exists(): + target.write_text('postgresql://messenger:'+values['postgres_password']+'@postgres:5432/messenger',encoding='utf-8') +env = root / '.env' +target = folder / 'broker_url' +if not target.exists(): + target.write_text('redis://:'+values['broker_password']+'@broker:6379/0',encoding='utf-8') +if not env.exists(): + env.write_text('LAB5_PORT=8085\nSESSION_TTL_SECONDS=3600\nWORKER_CONCURRENCY=2\nJOB_MAX_ATTEMPTS=5\nJOB_TIMEOUT_SECONDS=60\nJOB_LEASE_SECONDS=90\nSTAGING_RETENTION_SECONDS=86400\nMAX_PENDING_JOBS=1000\n',encoding='utf-8') +print('Local lab5 secrets prepared; existing files preserved. No secret values displayed.') diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..1d5ffcfd --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,5 @@ +@echo off +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api1" (docker compose --env-file .env stop -t 1 api1 & exit /b) +if "%TARGET_INSTANCE_ID%"=="api2" (docker compose --env-file .env stop -t 1 api2 & exit /b) +exit /b 2 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..645df985 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +case "${TARGET_INSTANCE_ID:-}" in + api1|api2) docker compose --env-file .env stop -t 1 "$TARGET_INSTANCE_ID" ;; + *) exit 2 ;; +esac diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..83103287 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,251 @@ +"""Explicit fault-injection campaign, scoped to the mtusi-lab5 Compose project.""" +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +os.environ['PYTHONDONTWRITEBYTECODE']='1' +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,Failure,check +EVIDENCE = ROOT/'evidence' +BASE = os.getenv('BASE_URL','http://127.0.0.1:8085') +DC = ['docker','compose','--env-file','.env','-f','compose.yaml','-f','compose.test.yaml'] +results = [] + + +def command(args,filename=None,allow_failure=False): + proc = subprocess.run(args,cwd=ROOT,encoding='utf-8',errors='replace',capture_output=True,timeout=180) + if filename: + (EVIDENCE/filename).write_text(proc.stdout+proc.stderr,encoding='utf-8') + if proc.returncode and not allow_failure: + raise RuntimeError('Command failed: '+' '.join(args)+'\n'+proc.stdout+proc.stderr) + return proc + + +def compose(*args): + return command(DC+list(args)) + + +def dbtool(*args,allow_failure=False): + proc = command(DC+['run','--rm','--no-deps','test-runner','python','student-tests/dbtool.py',*map(str,args)],allow_failure=allow_failure) + if allow_failure: + return proc.returncode + return json.loads(proc.stdout) + + +def save(name,value): + (EVIDENCE/name).write_text(json.dumps(value,ensure_ascii=False,indent=2),encoding='utf-8') + + +def wait_health(c): + deadline = time.monotonic()+90 + while time.monotonic()0: + break + check(time.monotonic()0,'broker lost unconsumed job') + c.call('GET','/api/v1/users/me',actor=alice,schema='User') + compose('start','worker') + _,info = ready(c,bob,att,data) + record('worker_off_api_broker_restart',started,{'queue_before':queue_before,'queue_after':queue_after,'before':before,'after':after,'job':info}) + + started = time.monotonic() + baseline = info + dbtool('duplicate',att['id'],1,12) + time.sleep(2) + repeated = dbtool('inspect',att['id']) + check(repeated['objects']==baseline['objects'],'successful duplicate rewrote objects') + check(repeated['jobs']==baseline['jobs'],'successful duplicate changed job') + check(any(e['event']=='duplicate_ignored' for e in repeated['events']),'duplicates not consumed') + record('12_duplicate_deliveries',started,{'job':repeated}) + + for point in ('before_publish','after_publish','before_effect','after_object','before_result_commit','after_result_commit'): + started = time.monotonic() + compose('stop','-t','1','worker','publisher') + att,data = c.attachment(alice,chat) + dbtool('fault',att['id'],point) + compose('start','publisher','worker') + _,info = ready(c,bob,att,data) + check(any(e['event']=='fault' and e['detail']==point for e in info['events']),'fault not triggered') + if point in ('before_effect','after_object','before_result_commit'): + check(any(e['event']=='lease_recovered' for e in info['events']),'dead worker lease not recovered') + check(info['jobs'][0]['attempts']==2,'crash attempt accounting') + if point=='after_result_commit': + deadline = time.monotonic()+40 + while not any(e['event']=='duplicate_ignored' for e in info['events']): + check(time.monotonic()=1,'invalid envelope registry') + stages = dbtool('staging') + check('staging/'+att['id'] not in stages['keys'],'ready staging not cleaned') + record('invalid_envelope_and_staging_cleanup',started,{'queue':queue,'staging_count':len(stages['keys'])}) + finally: + compose('start','api1','api2','broker','s3','postgres','publisher','worker') + save('before-restart.json',snapshot()) + suffix = '.cmd' if os.name=='nt' else '.sh' + command([sys.executable,'-X','utf8','tests/resilience.py','--base-url',BASE,'--mode','persistence','--action-script',str(ROOT/'scripts'/('restart'+suffix))],'persistence.txt') + save('after-restart.json',snapshot()) + print('PASS: public persistence',flush=True) + command([sys.executable,'-X','utf8','tests/resilience.py','--base-url',BASE,'--mode','failover','--action-script',str(ROOT/'scripts'/('stop-one'+suffix))],'failover.txt') + print('PASS: public failover',flush=True) + compose('start','api1','api2') + wait_health(c) + command([sys.executable,'-X','utf8','tests/smoke.py','--base-url',BASE],'final-smoke.txt') + save('final-containers.json',snapshot()) + save('verification.json',{'result':'PASS','pipeline_scenarios':len(results),'public_smoke':True,'public_persistence':True,'public_failover':True,'image_acceptance':True,'base_url':BASE}) + print('PASS: all lab5 verification',flush=True) + + +if __name__=='__main__': + main() diff --git a/scripts/verify_cleanup.py b/scripts/verify_cleanup.py new file mode 100644 index 00000000..0fa4e72a --- /dev/null +++ b/scripts/verify_cleanup.py @@ -0,0 +1,16 @@ +"""Explicit small retention test; shortened TTL is isolated to the test process.""" +from verify import BASE,ROOT,EVIDENCE,Client,compose,command,ready,save,DC + +client = Client(BASE,timeout=5) +alice,bob,eve,chat = client.bootstrap() +try: + compose('stop','-t','1','worker','publisher') + queued,data = client.attachment(alice,chat) + failed,_ = client.attachment(alice,chat) + proc = command(DC+['run','--rm','--no-deps','test-runner','python','student-tests/maintenance.py',queued['id'],failed['id']]) + (EVIDENCE/'staging-retention.txt').write_text(proc.stdout,encoding='utf-8') +finally: + compose('start','publisher','worker') +_,info = ready(client,bob,queued,data) +save('retention-active-recovery.json',{'result':'PASS','job':info}) +print('PASS: retention preserves active input, removes expired failed/orphan staging, recovery succeeds.') diff --git a/scripts/verify_redelivery.py b/scripts/verify_redelivery.py new file mode 100644 index 00000000..a0d54b08 --- /dev/null +++ b/scripts/verify_redelivery.py @@ -0,0 +1,21 @@ +"""Recheck the post-COMMIT/pre-ack fault after the full campaign has drained.""" +import json +import time +from verify import EVIDENCE,dbtool,save,check + +campaign = json.loads((EVIDENCE/'pipeline-results.json').read_text(encoding='utf-8')) +case = next(x for x in campaign['scenarios'] if x['scenario']=='after_result_commit') +deadline = time.monotonic()+40 +while True: + final = dbtool('inspect',case['job']['attachment_id']) + if any(e['event']=='duplicate_ignored' for e in final['events']): + break + check(time.monotonic()clock_timestamp()', (token_hash,)).fetchone() + if not row: + fail(401) + return row, token_hash + + +def access(conn, uid, cid): + row = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s', (cid,uid)).fetchone() + if not row: + fail(404) + return row + + +def authorize_upload(authorization, cid): + with db() as conn: + actor, _ = authenticate(conn, authorization) + cid = ident(cid) + access(conn, actor['id'], cid) + return str(actor['id']), cid + + +def query(request, allowed): + if set(request.query_params) - set(allowed) or any(len(request.query_params.getlist(k)) != 1 for k in request.query_params): + fail(400) + return dict(request.query_params) + + +def page(conn, kind, resource, actor, params): + raw = params.get('limit','50') + if len(raw)>3 or not re.fullmatch(r'[0-9]+', raw) or not 1 <= int(raw) <= 100: + fail(400) + limit = int(raw) + boundary = dt.datetime.min.replace(tzinfo=dt.timezone.utc) + bid = '00000000-0000-0000-0000-000000000000' + scope = kind + ':' + resource + if 'cursor' in params: + if len(params['cursor'])>2048: + fail(400) + try: + encoded, signature = params['cursor'].split('.') + expected = hmac.new(setting('CURSOR_SECRET').encode(), encoded.encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected): + fail(400) + payload = json.loads(base64.urlsafe_b64decode(encoded + '=' * (-len(encoded) % 4))) + if payload['scope'] != scope: + fail(400) + boundary = dt.datetime.fromisoformat(payload['time']) + bid = ident(payload['id']) + except (ValueError, KeyError, TypeError): + fail(400) + if kind == 'chats': + rows = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s::uuid) ORDER BY c.created_at,c.id LIMIT %s', (actor,boundary,bid,limit+1)).fetchall() + items = [chat(conn,x) for x in rows[:limit]] + else: + rows = conn.execute('SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s::uuid) ORDER BY created_at,id LIMIT %s', (resource,boundary,bid,limit+1)).fetchall() + items = [message(conn,x) for x in rows[:limit]] + cursor = None + if len(rows) > limit: + last = rows[limit-1] + encoded = base64.urlsafe_b64encode(json.dumps({'scope':scope,'time':stamp(last['created_at']),'id':str(last['id'])}, sort_keys=True).encode()).decode().rstrip('=') + cursor = encoded + '.' + hmac.new(setting('CURSOR_SECRET').encode(), encoded.encode(), hashlib.sha256).hexdigest() + return {'items':items, 'next_cursor':cursor} + + +@asynccontextmanager +async def lifespan(app): + app.state.upload_slots = asyncio.Semaphore(4) + app.state.json_slots = asyncio.Semaphore(16) + yield + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +def error_response(request, status): + headers = {} + if status == 401: + headers['WWW-Authenticate'] = ('Basic realm="messenger", charset="UTF-8"' if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + return JSONResponse({'error':{'code':CODES[status], 'message':MESSAGES[status], + 'request_id':request.state.request_id}}, status, headers=headers) + + +@app.middleware('http') +async def errors_and_identity(request, call_next): + request.state.request_id = uuid.uuid4().hex + try: + result = await call_next(request) + except APIError as exc: + result = error_response(request, exc.status) + except (psycopg.Error, BotoCoreError, ClientError) as exc: + code = exc.response.get('Error',{}).get('Code','') if isinstance(exc,ClientError) else getattr(exc,'sqlstate','') + logging.warning('Storage failure: %s code=%s',type(exc).__name__,code) + result = error_response(request, 503) + except Exception: + result = error_response(request, 500) + result.headers['X-Request-Id'] = request.state.request_id + result.headers['X-Instance-Id'] = INSTANCE + return result + + +@app.exception_handler(APIError) +async def api_error(request, exc): + return error_response(request, exc.status) + + +@app.exception_handler(HTTPException) +async def http_error(request, exc): + return error_response(request, 404 if exc.status_code == 404 else 400) + + +@app.get('/health/live') +def live(): + return {'status':'ok'} + + +@app.get('/internal-test/memory') +def test_memory(): + if os.getenv('ENABLE_TEST_METRICS') != '1': + fail(404) + with open('/proc/self/status') as status: + rss = next(int(line.split()[1])*1024 for line in status if line.startswith('VmRSS:')) + usage = os.times() + with open('/sys/fs/cgroup/memory.current') as current: + container_memory = int(current.read()) + with open('/sys/fs/cgroup/memory.peak') as peak: + container_peak = int(peak.read()) + return {'rss_bytes':rss, 'cpu_seconds':usage.user+usage.system, + 'container_memory_bytes':container_memory, 'container_peak_bytes':container_peak} + + +@app.get('/health/ready') +def ready(): + with db() as conn: + conn.execute('SELECT 1 FROM attachments LIMIT 1') + s3().head_bucket(Bucket=BUCKET) + return {'status':'ok'} + + +def validate_image(file, declared): + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + file.seek(0) + with Image.open(file) as im: + if im.format not in ('PNG','JPEG') or Image.MIME[im.format] != declared: + fail(415) + if im.width * im.height > MAX_PIXELS: + fail(413) + if getattr(im,'n_frames',1) != 1: + fail(415) + im.verify() + file.seek(0) + with Image.open(file) as im: + im.load() # JPEG verify() alone does not detect truncation. + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + fail(413) + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + fail(415) + file.seek(0) + + +def persist_upload(file, authorization, cid, filename, declared, size, sha): + validate_image(file, declared) + client = s3() + aid = str(uuid.uuid4()) + key = 'staging/' + aid + # Shared transaction lock covers PUT -> COMMIT; reconciler takes the exclusive lock. + # If COMMIT outcome is ambiguous, never delete here: the reconciler checks the committed DB. + with db() as conn: + actor, _ = authenticate(conn, authorization) + access(conn, actor['id'], cid) + conn.execute('SELECT pg_advisory_xact_lock_shared(%s)', (STORAGE_LOCK,)) + # Serialize admission across replicas; the DB includes unpublished work. + conn.execute('SELECT pg_advisory_xact_lock(550005)') + pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE status IN ('queued','processing')").fetchone()['n'] + if pending >= int(os.getenv('MAX_PENDING_JOBS','1000')): + fail(429) + md5 = base64.b64encode(hashlib.file_digest(file, 'md5').digest()).decode() + file.seek(0) + client.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=size, + ContentType=declared, Metadata={'sha256':sha}, + ContentMD5=md5) + # Test-only fault injection is disabled in the regular Compose configuration. + if os.getenv('FAULT_AFTER_PUT') == '1': + fail(503) + row = conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,staging_key,content_type,size_bytes,sha256) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', + (aid,cid,actor['id'],filename,f'results/{aid}/v1/original',key,declared,size,sha)).fetchone() + enqueue(conn,aid,1) + result = attachment(row) + return result + + +@app.post('/api/v1/chats/{cid}/attachments') +async def upload(request: Request, cid: str): + authorization = request.headers.get('authorization','') + _, cid = await run_in_threadpool(authorize_upload, authorization, cid) + params = query(request, ('filename',)) + filename = string(params.get('filename'),1,255) + if any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename): + fail(400) + declared = request.headers.get('content-type','').lower() + if declared not in ('image/png','image/jpeg'): + fail(415) + length = request.headers.get('content-length') + if length is not None: + if len(length)>10 or not re.fullmatch('[0-9]+', length): + fail(400) + if int(length) > MAX_BYTES: + fail(413) + try: + await asyncio.wait_for(request.app.state.upload_slots.acquire(), timeout=1) + except TimeoutError: + fail(429) + try: + with tempfile.TemporaryFile() as file: # receive buffer; durable staging is S3 before 202 + digest = hashlib.sha256() + size = 0 + async def receive(): + nonlocal size + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + fail(413) + digest.update(chunk) + file.write(chunk) + try: + await asyncio.wait_for(receive(), timeout=30) + except (TimeoutError, ClientDisconnect): + fail(400) + if not size or (length is not None and size != int(length)): + fail(415 if not size else 400) + file.seek(0) + # file_digest leaves position at EOF; compute MD5 without moving the PUT body. + result = await run_in_threadpool(persist_upload, file, authorization, cid, filename, declared, size, digest.hexdigest()) + return JSONResponse(result,202) + finally: + request.app.state.upload_slots.release() + + +def dispatch(method, path, authorization, data, params, headers): + with db() as conn: + if path == '/api/v1/users' and method == 'POST': + value = strict_json(data, ('username','display_name','password')) + name = string(value['username'],3,32) + if not re.fullmatch('[a-z0-9_]{3,32}',name): + fail(400) + display = string(value['display_name'],1,100) + password = string(value['password'],12,128,False) + try: + row = conn.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s) RETURNING *', + (str(uuid.uuid4()),name,display,PH.hash(password))).fetchone() + except psycopg.errors.UniqueViolation: + fail(409) + return 201, user(row) + if path == '/api/v1/auth/sessions' and method == 'POST': + if data or not authorization.startswith('Basic '): + fail(401) + try: + name,password = base64.b64decode(authorization[6:],validate=True).decode('utf-8').split(':',1) + except (ValueError, UnicodeError): + fail(401) + row = conn.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone() + try: + PH.verify(row['password_hash'] if row else DUMMY_HASH, password) + except VerificationError: + fail(401) + if not row: + fail(401) + token = secrets.token_urlsafe(32) + expires = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=int(os.getenv('SESSION_TTL_SECONDS','3600'))) + conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)',(hashlib.sha256(token.encode()).hexdigest(),row['id'],expires)) + return 201, {'token':token,'token_type':'Bearer','expires_at':stamp(expires),'user':user(row)} + actor, token_hash = authenticate(conn, authorization) + uid = actor['id'] + if path == '/api/v1/users/me' and method == 'GET': + return 200, user(actor) + if path == '/api/v1/auth/sessions/current' and method == 'DELETE': + conn.execute('DELETE FROM sessions WHERE token_hash=%s',(token_hash,)) + return 204, None + if path == '/api/v1/chats': + if method == 'POST': + value = strict_json(data,('title','member_ids')) + title = string(value['title'],1,100) + members = ids(value['member_ids'],100,1) + if str(uid) not in members or conn.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(members,)).fetchone()['n'] != len(members): + fail(400) + row = conn.execute('INSERT INTO chats(id,title) VALUES(%s,%s) RETURNING *',(str(uuid.uuid4()),title)).fetchone() + for member in members: + conn.execute('INSERT INTO members VALUES(%s,%s)',(row['id'],member)) + return 201, chat(conn,row) + if method == 'GET': + return 200, page(conn,'chats',str(uid),uid,params) + match = re.fullmatch(r'/api/v1/chats/([^/]+)(/messages)?', path) + if match: + cid = ident(match[1]) + current_chat = access(conn,uid,cid) + if not match[2] and method == 'GET': + return 200, chat(conn,current_chat) + if match[2] and method == 'GET': + return 200, page(conn,'messages',cid,uid,params) + if match[2] and method == 'POST': + value = strict_json(data,('text',),('attachment_ids',)) + text = string(value['text'],0,4000,False) + attachments = ids(value.get('attachment_ids',[]),10) + if not text.strip() and not attachments: + fail(400) + key = headers.get('idempotency-key','') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key): + fail(400) + canonical = json.dumps({'text':text,'attachment_ids':attachments},sort_keys=True,ensure_ascii=False) + body_hash = hashlib.sha256(canonical.encode()).hexdigest() + # One transaction lock per user/chat/key, shared across API replicas. + lock = int.from_bytes(hashlib.sha256((str(uid)+cid+key).encode()).digest()[:8],'big',signed=True) + conn.execute('SELECT pg_advisory_xact_lock(%s)',(lock,)) + for aid in attachments: + a = conn.execute('SELECT * FROM attachments WHERE id=%s AND chat_id=%s',(aid,cid)).fetchone() + if not a: + fail(404) + if a['status'] == 'failed': + fail(409) + if a['availability'] != 'present': + fail(503) + old = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()', (uid,cid,key)).fetchone() + if old: + if old['body_hash'] != body_hash: + fail(409) + return 201, message(conn,conn.execute('SELECT * FROM messages WHERE id=%s',(old['message_id'],)).fetchone()) + conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s',(uid,cid,key)) + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES(%s,%s,%s,%s) RETURNING *',(str(uuid.uuid4()),cid,uid,text)).fetchone() + for pos,aid in enumerate(attachments): + conn.execute('INSERT INTO message_attachments VALUES(%s,%s,%s)',(row['id'],aid,pos)) + conn.execute("INSERT INTO idempotency VALUES(%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",(uid,cid,key,body_hash,row['id'])) + return 201, message(conn,row) + match = re.fullmatch(r'/api/v1/attachments/([^/]+)(/content)?',path) + if match and method == 'GET': + aid = ident(match[1]) + row = conn.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,uid)).fetchone() + if not row: + fail(404) + if not match[2]: + return 200, attachment(row) + if row['status'] != 'ready': + fail(409) + variant = params.get('variant','original') + selected = row if variant == 'original' else next((x for x in row['variants'] if x['name']==variant),None) + if selected is None: + fail(404) + if row['availability'] != 'present': + fail(503) + content = s3().get_object(Bucket=BUCKET,Key=selected['object_key']) + body = content['Body'] + def stream(): + try: + while chunk := body.read(65536): + yield chunk + finally: + body.close() + return 200, StreamingResponse(stream(),media_type=selected['content_type'], + headers={'Content-Length':str(selected['size_bytes']), 'X-Content-Type-Options':'nosniff'}) + fail(404) + + +@app.api_route('/api/v1/{path:path}', methods=['GET','POST','DELETE','PUT','PATCH']) +async def api(request: Request, path: str): + if not (path == 'users' and request.method == 'POST') and not (path == 'auth/sessions' and request.method == 'POST'): + def check_auth(): + with db() as conn: + authenticate(conn,request.headers.get('authorization','')) + await run_in_threadpool(check_auth) + params = query(request, ('limit','cursor') if request.method == 'GET' and (path == 'chats' or path.endswith('/messages')) else ('variant',) if path.endswith('/content') else ()) + try: + await asyncio.wait_for(request.app.state.json_slots.acquire(),timeout=1) + except TimeoutError: + fail(429) + try: + data = bytearray() + async def receive(): + async for chunk in request.stream(): + data.extend(chunk) + if len(data) > 65536: + fail(400) + try: + await asyncio.wait_for(receive(),timeout=10) + except (TimeoutError, ClientDisconnect): + fail(400) + # Authentication for protected routes is performed before parsing JSON in dispatch. + status, result = await run_in_threadpool(dispatch, request.method, '/api/v1/'+path, + request.headers.get('authorization',''),bytes(data),params,dict(request.headers)) + if isinstance(result, Response): + return result + return Response(status_code=204) if status == 204 else JSONResponse(result,status) + finally: + request.app.state.json_slots.release() diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 00000000..05217b6c --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,39 @@ +"""Bounded staging janitor. Never expires queued/processing originals.""" +import logging +import os +import time +from .storage import BUCKET,STORAGE_LOCK,db,s3 +from .jobs import lock_id + + +def sweep(): + client = s3() + retention = int(os.getenv('STAGING_RETENTION_SECONDS','86400')) + count = 0 + for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET,Prefix='staging/',PaginationConfig={'PageSize':100}): + for obj in page.get('Contents',[]): + if time.time()-obj['LastModified'].timestamp()=retention: + client.delete_object(Bucket=BUCKET,Key=obj['Key']) + count += 1 + return count + + +if __name__=='__main__': + while True: + try: + print('Expired staging deleted:',sweep(),flush=True) + except Exception as exc: + logging.warning('Cleanup aborted: %s',type(exc).__name__) + time.sleep(60) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..3653c4f3 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,78 @@ +"""DB is the durable intent; Redis Streams provides at-least-once notifications.""" +import datetime as dt +import hashlib +import json +import os +import uuid +import redis +from psycopg.types.json import Jsonb +from .storage import setting + +STREAM = 'attachments:v1' +GROUP = 'image-workers' +MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS','5')) +TIMEOUT = int(os.getenv('JOB_TIMEOUT_SECONDS','60')) +LEASE = max(int(os.getenv('JOB_LEASE_SECONDS','90')), TIMEOUT + 5) + + +def broker(): + return redis.Redis.from_url(setting('BROKER_URL'),decode_responses=True, + socket_timeout=5, socket_connect_timeout=3) + + +def lock_id(aid): + return int.from_bytes(hashlib.sha256(str(aid).encode()).digest()[:8],'big',signed=True) + + +def audit(conn,jid,event,detail=''): + conn.execute('INSERT INTO job_audit(job_id,event,detail) VALUES(%s,%s,%s)',(jid,event,detail)) + + +def enqueue(conn,aid,version): + jid = uuid.uuid4() + conn.execute('INSERT INTO jobs(id,attachment_id,pipeline_version) VALUES(%s,%s,%s)',(jid,aid,version)) + conn.execute('INSERT INTO outbox(job_id) VALUES(%s)',(jid,)) + audit(conn,jid,'accepted') + return jid + + +def payload(row): + return {'schema_version':1,'job_id':str(row['id']), 'type':'attachment.process.v1', + 'attachment_id':str(row['attachment_id']),'pipeline_version':row['pipeline_version'], + 'attempt':row['attempts']+1, + 'created_at':row['created_at'].astimezone(dt.timezone.utc).isoformat().replace('+00:00','Z')} + + +def parse_payload(raw): + value = json.loads(raw) + required = {'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'} + if not isinstance(value,dict) or not required <= value.keys() or value.keys()-required-{'traceparent','tracestate'}: + raise ValueError('invalid envelope') + if value['schema_version'] != 1 or value['type'] != 'attachment.process.v1': + raise ValueError('invalid envelope') + for key in ('job_id','attachment_id'): + uuid.UUID(value[key]) + for key in ('pipeline_version','attempt'): + if type(value[key]) is not int or value[key] < 1: + raise ValueError('invalid envelope') + dt.datetime.fromisoformat(value['created_at'].replace('Z','+00:00')) + return value + + +def trip(conn,jid,point): + if os.getenv('ENABLE_FAULTS') != '1': + return False + row = conn.execute('UPDATE fault_rules SET remaining=remaining-1 WHERE job_id=%s AND point=%s AND remaining>0 RETURNING job_id',(jid,point)).fetchone() + if row: + audit(conn,jid,'fault',point) + return bool(row) + + +def finish_failed(conn,job,code): + conn.execute("UPDATE jobs SET status='failed',error_code=%s,lease_until=NULL WHERE id=%s",(code,job['id'])) + conn.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",(code,job['attachment_id'],job['pipeline_version'])) + audit(conn,job['id'],'failed',code) + + +def set_variants(conn,aid,version,variants): + conn.execute("UPDATE attachments SET variants=%s,status='ready',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",(Jsonb(variants),aid,version)) diff --git a/src/operator.py b/src/operator.py new file mode 100644 index 00000000..db581830 --- /dev/null +++ b/src/operator.py @@ -0,0 +1,58 @@ +"""Explicit operator-only CLI. No public replay/version/fault HTTP endpoints.""" +import argparse +import json +from .storage import BUCKET,db,s3 +from .jobs import audit,enqueue,lock_id,trip + + +def replay(aid): + with db() as conn: + conn.execute('SELECT pg_advisory_xact_lock(%s)',(lock_id(aid),)) + row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(aid,)).fetchone() + if not row or row['status']!='failed': + raise ValueError('replay requires failed attachment') + # Do not reset successful jobs; only replay if immutable input still exists. + head = s3().head_object(Bucket=BUCKET,Key=row['staging_key']) + if head['ContentLength']!=row['size_bytes'] or head['Metadata'].get('sha256')!=row['sha256']: + raise ValueError('input unavailable') + job = conn.execute('SELECT * FROM jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE',(aid,row['pipeline_version'])).fetchone() + conn.execute("UPDATE jobs SET status='queued',attempts=0,error_code=NULL,lease_until=NULL,next_attempt_at=clock_timestamp() WHERE id=%s",(job['id'],)) + conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(aid,)) + conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],)) + audit(conn,job['id'],'operator_replay') + return str(job['id']) + + +def upgrade(aid): + with db() as conn: + conn.execute('SELECT pg_advisory_xact_lock(%s)',(lock_id(aid),)) + row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(aid,)).fetchone() + if not row or row['status']!='ready': + raise ValueError('upgrade requires ready attachment') + client = s3() + client.copy_object(Bucket=BUCKET,Key=row['staging_key'],CopySource={'Bucket':BUCKET,'Key':row['object_key']}) + version = row['pipeline_version']+1 + conn.execute("UPDATE jobs SET status='superseded' WHERE attachment_id=%s",(aid,)) + conn.execute("UPDATE attachments SET pipeline_version=%s,object_key=%s,status='queued',variants='[]',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(version,f'results/{aid}/v{version}/original',aid)) + jid = enqueue(conn,aid,version) + audit(conn,jid,'pipeline_upgrade') + return str(jid) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action',choices=['failed','replay','upgrade']) + parser.add_argument('attachment_id',nargs='?') + args = parser.parse_args() + if args.action=='failed': + with db() as conn: + rows = conn.execute("SELECT id,attachment_id,pipeline_version,attempts,error_code FROM jobs WHERE status='failed' ORDER BY created_at LIMIT 100").fetchall() + print(json.dumps(rows,default=str)) + else: + if not args.attachment_id: + parser.error('attachment_id required') + print((replay if args.action=='replay' else upgrade)(args.attachment_id)) + + +if __name__=='__main__': + main() diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 00000000..cd70489e --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,64 @@ +"""Outbox relay and lease recovery. A lost publish response causes a safe duplicate.""" +import json +import logging +import os +import time +from .storage import db +from .jobs import broker, STREAM, MAX_ATTEMPTS, lock_id, payload, audit, trip, finish_failed + + +def recover(): + with db() as conn: + rows = conn.execute("SELECT * FROM jobs WHERE status='processing' AND lease_until= MAX_ATTEMPTS: + finish_failed(conn,job,'retry_exhausted') + else: + conn.execute("UPDATE jobs SET status='queued',lease_until=NULL,next_attempt_at=clock_timestamp() WHERE id=%s",(job['id'],)) + conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",(job['attachment_id'],job['pipeline_version'])) + conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],)) + audit(conn,job['id'],'lease_recovered') + + +def publish_once(): + with db() as conn: + row = conn.execute("""SELECT j.* FROM jobs j JOIN outbox o ON o.job_id=j.id + WHERE j.status='queued' AND j.next_attempt_at<=clock_timestamp() + AND (o.published_at IS NULL OR o.published_atImage.MAX_IMAGE_PIXELS or getattr(original,'n_frames',1)!=1: + raise ValueError('invalid image') + oriented = ImageOps.exif_transpose(original) + # Composite transparency before resize; strip all metadata by fresh RGB image. + rgba = oriented.convert('RGBA') + rgb = Image.new('RGB',rgba.size,'white') + rgb.paste(rgba,mask=rgba.getchannel('A')) + rgba.close() + oriented.close() + sizes = {} + for name, bound, fmt, quality in [('thumbnail',256,'JPEG',85),('crop',128,'JPEG',85),('optimized',1280,'WEBP',80)]: + image = ImageOps.fit(rgb,(128,128),method=Image.Resampling.LANCZOS,centering=(0.5,0.5)) if name=='crop' else rgb.resize(fit_size(rgb.size,bound),Image.Resampling.LANCZOS) + image.info.clear() + image.save(Path(destination)/name,format=fmt,quality=quality,**({'method':6} if fmt=='WEBP' else {'optimize':True,'subsampling':0})) + sizes[name] = image.size + image.close() + rgb.close() + return sizes + + +def main(): + resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024)) + timeout = int(os.getenv('JOB_TIMEOUT_SECONDS','60')) + resource.setrlimit(resource.RLIMIT_CPU,(timeout,timeout+1)) + try: + result = {'sizes':transform(sys.argv[1],sys.argv[2])} + except MemoryError: + result = {'error':'image_resource_limit'} + except (ValueError,OSError,UnidentifiedImageError,Image.DecompressionBombError,Image.DecompressionBombWarning): + result = {'error':'invalid_image'} + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..84bef10e --- /dev/null +++ b/src/worker.py @@ -0,0 +1,223 @@ +"""One notification per consumer; DB lease and session lock fence all side effects.""" +import hashlib +import json +import logging +import multiprocessing +import os +from pathlib import Path +import random +import signal +import subprocess +import sys +import tempfile +import time +import uuid +from botocore.exceptions import ClientError +import redis +from .storage import BUCKET, db, s3 +from .jobs import (STREAM,GROUP,LEASE,TIMEOUT,MAX_ATTEMPTS,broker,lock_id, + parse_payload,audit,trip,finish_failed,set_variants) + + +class Permanent(Exception): + pass + + +def verify_ready(client,row): + for effect in [row]+row['variants']: + head = client.head_object(Bucket=BUCKET,Key=effect['object_key']) + if head['ContentLength'] != effect['size_bytes'] or head.get('Metadata',{}).get('sha256') != effect['sha256']: + raise RuntimeError('effect unavailable') + + +def cleanup(client,row): + if row['staging_key']: + client.delete_object(Bucket=BUCKET,Key=row['staging_key']) + + +def crash(conn,jid,point): + with conn.transaction(): + fire = trip(conn,jid,point) + if fire: + os._exit(93) + + +def process(value): + with db() as conn: + conn.autocommit = True + # Held across claim, decode, PUT and COMMIT. A dead process releases it. + if not conn.execute('SELECT pg_try_advisory_lock(%s) AS ok',(lock_id(value['attachment_id']),)).fetchone()['ok']: + return True # active work is durable in DB; relay recovers its lease if needed + with conn.transaction(): + job = conn.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE',(value['job_id'],)).fetchone() + if not job: + return True + if str(job['attachment_id']) != value['attachment_id'] or job['pipeline_version'] != value['pipeline_version']: + return True + row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone() + if row['pipeline_version'] != job['pipeline_version'] or job['status']=='superseded': + audit(conn,job['id'],'stale_ignored') + return True + if job['status']=='ready': + verify_ready(s3(),row) + cleanup(s3(),row) + audit(conn,job['id'],'duplicate_ignored') + return True + if job['status']=='failed': + return True + # Retry notifications are only valid for the next managed attempt. + if job['status']!='queued' or value['attempt']!=job['attempts']+1: + return True + due = conn.execute('SELECT next_attempt_at<=clock_timestamp() AS ok FROM jobs WHERE id=%s',(job['id'],)).fetchone()['ok'] + if not due: + return True + if job['attempts']>=MAX_ATTEMPTS: + finish_failed(conn,job,'retry_exhausted') + return True + job = conn.execute("UPDATE jobs SET status='processing',attempts=attempts+1,lease_until=clock_timestamp()+(%s*interval '1 second') WHERE id=%s RETURNING *",(LEASE,job['id'])).fetchone() + conn.execute("UPDATE attachments SET status='processing',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + audit(conn,job['id'],'processing',str(job['attempts'])) + client = s3() + try: + crash(conn,job['id'],'before_effect') + with conn.transaction(): + transient = trip(conn,job['id'],'transient') + permanent = trip(conn,job['id'],'permanent') + if transient: + raise TimeoutError() + if permanent: + raise Permanent('invalid_image') + def timeout_handler(signum,frame): + raise TimeoutError() + signal.signal(signal.SIGALRM,timeout_handler) + signal.alarm(TIMEOUT) + with conn.transaction(): + hang = trip(conn,job['id'],'hang') + if hang: + time.sleep(TIMEOUT+2) # test exercises the real wall-clock deadline + with tempfile.TemporaryDirectory() as folder: + source = Path(folder)/'original' + response = client.get_object(Bucket=BUCKET,Key=row['staging_key']) + digest = hashlib.sha256() + size = 0 + try: + with source.open('wb') as file: + for chunk in response['Body'].iter_chunks(65536): + size += len(chunk) + if size>10485760: + raise Permanent('invalid_image') + digest.update(chunk) + file.write(chunk) + finally: + response['Body'].close() + if size!=row['size_bytes'] or digest.hexdigest()!=row['sha256']: + raise Permanent('checksum_mismatch') + child = subprocess.Popen([sys.executable,'-m','src.transform',str(source),folder],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True) + try: + output,_ = child.communicate(timeout=TIMEOUT) + finally: + if child.poll() is None: + child.kill() + child.wait() + if child.returncode: + raise Permanent('image_resource_limit') + transformed = json.loads(output) + if 'error' in transformed: + raise Permanent(transformed['error']) + variants = [] + for name in ('original','thumbnail','crop','optimized'): + file = Path(folder)/name + data = file.read_bytes() + sha = hashlib.sha256(data).hexdigest() + key = f"results/{row['id']}/v{job['pipeline_version']}/{name}" + mime = row['content_type'] if name=='original' else 'image/webp' if name=='optimized' else 'image/jpeg' + client.put_object(Bucket=BUCKET,Key=key,Body=data,ContentType=mime,Metadata={'sha256':sha}) + if name=='original': + crash(conn,job['id'],'after_object') + else: + width,height = transformed['sizes'][name] + variants.append({'name':name,'object_key':key,'content_type':mime,'size_bytes':len(data),'sha256':sha,'width':width,'height':height}) + crash(conn,job['id'],'before_result_commit') + with conn.transaction(): + set_variants(conn,row['id'],job['pipeline_version'],variants) + conn.execute("UPDATE jobs SET status='ready',lease_until=NULL,error_code=NULL WHERE id=%s",(job['id'],)) + audit(conn,job['id'],'ready') + crash(conn,job['id'],'after_result_commit') + signal.alarm(0) + cleanup(client,row) + return True + except Exception as exc: + signal.alarm(0) + with conn.transaction(): + latest = conn.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE',(job['id'],)).fetchone() + # A cleanup/network failure after ready never rolls back a committed result. + if latest['status']=='ready': + return False + permanent = isinstance(exc,Permanent) + if isinstance(exc,ClientError) and exc.response.get('Error',{}).get('Code')=='NoSuchKey': + permanent,exc = True,Permanent('staging_missing') + if permanent or job['attempts']>=MAX_ATTEMPTS: + finish_failed(conn,job,str(exc) if permanent else 'retry_exhausted') + else: + base = float(os.getenv('RETRY_BASE_SECONDS','1')) + delay = min(30,base*2**(job['attempts']-1))*random.uniform(0.5,1.5) + conn.execute("UPDATE jobs SET status='queued',lease_until=NULL,error_code='temporary_failure',next_attempt_at=clock_timestamp()+(%s*interval '1 second') WHERE id=%s",(delay,job['id'])) + conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],)) + audit(conn,job['id'],'retry_scheduled',f'{delay:.3f}') + return True + + +def consumer(): + name = uuid.uuid4().hex + cursor = '0-0' + while True: + try: + r = broker() + try: + r.xgroup_create(STREAM,GROUP,id='0-0',mkstream=True) + except redis.ResponseError as exc: + if 'BUSYGROUP' not in str(exc): + raise + cursor,entries,_ = r.xautoclaim(STREAM,GROUP,name,LEASE*1000,cursor,count=1) + if not entries: + result = r.xreadgroup(GROUP,name,{STREAM:'>'},count=1,block=1000) + entries = result[0][1] if result else [] + for mid,fields in entries: + try: + value = parse_payload(fields.get('job','')) + except (ValueError,TypeError,KeyError): + r.xadd('attachments:invalid',{'message_id':mid,'error_code':'invalid_envelope'}) + r.xack(STREAM,GROUP,mid) + r.xdel(STREAM,mid) + continue + if process(value): + r.xack(STREAM,GROUP,mid) + r.xdel(STREAM,mid) # only one consumer group; no pending work is trimmed + except Exception as exc: + logging.warning('Consumer unavailable: %s',type(exc).__name__) + time.sleep(1) + + +def main(): + count = int(os.getenv('WORKER_CONCURRENCY','2')) + if not 1<=count<=8: + raise ValueError('WORKER_CONCURRENCY must be 1..8') + processes = [] + while True: + alive = [] + for p in processes: + if p.is_alive(): + alive.append(p) + else: + p.join() + processes = alive + while len(processes)2 else None + if action=='inspect': + return inspect(aid) + if action in ('replay','upgrade'): + return {'job_id':(replay if action=='replay' else upgrade)(aid)} + if action=='fault': + assert os.getenv('ENABLE_FAULTS')=='1' + with db() as conn: + job = conn.execute('SELECT * FROM jobs WHERE attachment_id=%s ORDER BY pipeline_version DESC LIMIT 1',(aid,)).fetchone() + conn.execute('INSERT INTO fault_rules(job_id,point,remaining) VALUES(%s,%s,%s) ON CONFLICT(job_id) DO UPDATE SET point=excluded.point,remaining=excluded.remaining',(job['id'],sys.argv[3],int(sys.argv[4]) if len(sys.argv)>4 else 1)) + return {'job_id':str(job['id']),'point':sys.argv[3]} + if action=='duplicate': + with db() as conn: + job = conn.execute('SELECT * FROM jobs WHERE attachment_id=%s AND pipeline_version=%s',(aid,int(sys.argv[3]))).fetchone() + value = payload(job) + value['attempt'] = 1 + ids = [broker().xadd(STREAM,{'job':json.dumps(value)}) for _ in range(int(sys.argv[4]))] + return {'job_id':str(job['id']),'notification_ids':ids} + if action=='queue': + r = broker() + return {'length':r.xlen(STREAM),'aof':r.config_get('appendonly'),'fsync':r.config_get('appendfsync'),'invalid':r.xlen('attachments:invalid')} + if action=='invalid': + broker().xadd(STREAM,{'job':'{"schema_version":99}'}) + return {'injected':True} + if action=='staging': + return {'keys':[x['Key'] for x in s3().list_objects_v2(Bucket=BUCKET,Prefix='staging/').get('Contents',[])]} + raise ValueError('unknown action') + + +if __name__=='__main__': + print(json.dumps(main(),default=str)) diff --git a/student-tests/maintenance.py b/student-tests/maintenance.py new file mode 100644 index 00000000..ea954450 --- /dev/null +++ b/student-tests/maintenance.py @@ -0,0 +1,45 @@ +"""Exercise staging retention while producer/worker are explicitly stopped.""" +import hashlib +import json +import os +from pathlib import Path +import sys +import time +import uuid +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from src.storage import BUCKET,db,s3 +from src.cleanup import sweep +from src.operator import replay + + +def main(): + queued,failed = sys.argv[1:3] + client = s3() + with db() as conn: + rows = conn.execute('SELECT * FROM attachments WHERE id=ANY(%s::uuid[])',([queued,failed],)).fetchall() + assert len(rows)==2 and all(x['status']=='queued' for x in rows) + conn.execute("UPDATE attachments SET status='failed',error_code='invalid_image',updated_at=clock_timestamp()-interval '10 seconds' WHERE id=%s",(failed,)) + conn.execute("UPDATE jobs SET status='failed',error_code='invalid_image' WHERE attachment_id=%s",(failed,)) + old_key = 'staging/'+str(uuid.uuid4()) + client.put_object(Bucket=BUCKET,Key=old_key,Body=b'orphan') + time.sleep(2) + fresh_key = 'staging/'+str(uuid.uuid4()) + client.put_object(Bucket=BUCKET,Key=fresh_key,Body=b'fresh') + os.environ['STAGING_RETENTION_SECONDS']='1' + removed = sweep() + keys = {x['Key'] for x in client.list_objects_v2(Bucket=BUCKET,Prefix='staging/').get('Contents',[])} + assert 'staging/'+queued in keys + assert 'staging/'+failed not in keys and old_key not in keys and fresh_key in keys + replay_denied = False + try: + replay(failed) + except Exception: + replay_denied = True + assert replay_denied + client.delete_object(Bucket=BUCKET,Key=fresh_key) + print(json.dumps({'result':'PASS','removed':removed,'active_preserved':queued,'expired_failed':failed, + 'old_orphan_removed':True,'fresh_orphan_preserved':True,'expired_replay_denied':True})) + + +if __name__=='__main__': + main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..aa329491 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,108 @@ +"""API/image acceptance in a private container; all users and images synthetic.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import io +import json +from pathlib import Path +import struct +import sys +import uuid +import zlib +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from PIL import Image +from client import Client,check +from src.storage import BUCKET,s3,setting + + +def image(mode,size,fmt,**kwargs): + stream = io.BytesIO() + im = Image.new(mode,size,(0,120,200,0) if mode=='RGBA' else (10,80,160)) + im.save(stream,format=fmt,**kwargs) + return stream.getvalue() + + +def main(): + c = Client() + alice,bob,eve,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id']+'/attachments?filename=test.png' + tested = [] + cases = [('fixture',(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes(),'image/png',(256,160)), + ('tiny_alpha',image('RGBA',(17,9),'PNG'),'image/png',(17,9)), + ('portrait',image('RGB',(199,401),'JPEG'),'image/jpeg',(127,256))] + exif = Image.Exif() + exif[274] = 6 + cases.append(('exif',image('RGB',(320,200),'JPEG',exif=exif),'image/jpeg',(160,256))) + for name,data,mime,expected in cases: + att = c.call('POST',path,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'] + ready = c.wait_ready(bob,att) + original = c.call('GET',f"/api/v1/attachments/{att['id']}/content",actor=bob)['bytes'] + check(data==original,'original bytes changed') + check({v['name'] for v in ready['variants']}=={'thumbnail','crop','optimized'},'incomplete variants') + for v in ready['variants']: + response = c.call('GET',f"/api/v1/attachments/{att['id']}/content?variant={v['name']}",actor=bob) + check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],'variant checksum') + with Image.open(io.BytesIO(response['bytes'])) as im: + check(im.size==(v['width'],v['height']),'actual dimensions') + check(not im.getexif() and not im.info.get('icc_profile'),'metadata not stripped') + if v['name']=='thumbnail': + check(im.size==expected,'floor/no upscale/orientation rule') + if name=='tiny_alpha': + check(min(im.getpixel((0,0)))>=250,'alpha not composited white') + if v['name']=='crop': + check(im.size==(128,128),'crop size') + if v['name']=='optimized': + check(im.format=='WEBP' and max(im.size)<=1280,'optimized WebP') + c.call('GET',f"/api/v1/attachments/{att['id']}/content?variant=missing",404,actor=alice) + tested.append(name) + invalid = [(b'broken','image/png',415),(cases[0][1],'image/jpeg',415),(image('RGB',(4,4),'GIF'),'image/png',415), + (cases[0][1][:-50],'image/png',415),(cases[2][1][:-20],'image/jpeg',415), + (b'a'*(10485760+1),'image/png',413)] + # Valid PNG signature/IHDR with huge dimensions; pixel check precedes decode. + ihdr = struct.pack('>IIBBBBB',40001,1000,8,2,0,0,0) + def chunk(kind,data): + return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)) + huge = b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',ihdr)+chunk(b'IDAT',zlib.compress(b'\x00'))+chunk(b'IEND',b'') + invalid.append((huge,'image/png',413)) + for data,mime,status in invalid: + c.call('POST',path,status,actor=alice,raw=data,headers={'Content-Type':mime}) + c.call('POST',path,404,actor=eve,raw=cases[0][1],headers={'Content-Type':'image/png'}) + for filename in ('../file.png','a%5Cb.png','a%00.png','a%0A.png'): + c.call('POST',path.split('?')[0]+'?filename='+filename,400,actor=alice,raw=cases[0][1],headers={'Content-Type':'image/png'}) + key = uuid.uuid4().hex + def post(_): + return Client().message(alice,chat,'concurrent',key=key)['json']['id'] + with ThreadPoolExecutor(max_workers=12) as pool: + ids = list(pool.map(post,range(12))) + check(len(set(ids))==1,'concurrent message idempotency') + # API credentials cannot write final results or delete staging. + from botocore.exceptions import ClientError + from src.storage import s3 as client_factory + import os + original_key,original_secret = os.environ['S3_ACCESS_KEY_ID_FILE'],os.environ['S3_SECRET_ACCESS_KEY_FILE'] + os.environ['S3_ACCESS_KEY_ID_FILE']='/run/secrets/s3_access_key_id' + os.environ['S3_SECRET_ACCESS_KEY_FILE']='/run/secrets/s3_secret_access_key' + api_client = client_factory() + denied = 0 + for op in (lambda:api_client.put_object(Bucket=BUCKET,Key='results/forbidden',Body=b'x'), + lambda:api_client.delete_object(Bucket=BUCKET,Key='staging/forbidden')): + try: + op() + except ClientError as exc: + check(exc.response['Error']['Code']=='AccessDenied','unexpected policy failure') + denied += 1 + os.environ['S3_ACCESS_KEY_ID_FILE'],os.environ['S3_SECRET_ACCESS_KEY_FILE']=original_key,original_secret + check(denied==2,'API privilege scope') + import urllib.error,urllib.request + try: + urllib.request.urlopen('http://s3:9000/'+BUCKET+'/results/forbidden',timeout=5) + raise AssertionError('anonymous access allowed') + except urllib.error.HTTPError as exc: + check(exc.code==403,'bucket not private') + summary = {'result':'PASS','images':tested,'negative_uploads':len(invalid)+5, + 'concurrent_repeats':12,'message_ids':len(set(ids)),'denied_s3_operations':denied} + print(json.dumps(summary)) + + +if __name__=='__main__': + main() -- 2.54.0