From 4ba890262724a4b16a28a7ec53f85a841899b5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BB=D0=B5=D0=B1=20=D0=9F=D0=B0=D1=80=D0=B0=D0=BC?= =?UTF-8?q?=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Fri, 16 Oct 2026 22:24:16 +0300 Subject: [PATCH 1/3] Implement durable versioned image pipeline and verify crash recovery --- .gitattributes | 3 + .github/pull_request_template.md | 27 +- ARCHITECTURE.md | 18 + Dockerfile | 16 + REPORT.md | 93 +++- checks/test_render.py | 49 ++ compose.yaml | 142 ++++++ evidence/integration.json | 177 +++++++ evidence/restart-processes.json | 759 +++++++++++++++++++++++++++++++ evidence/stop-one.json | 11 + infra/haproxy.cfg | 30 ++ migrations/001_initial.sql | 35 ++ migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 25 + migrations/004_jobs.sql | 41 ++ requirements-test.txt | 3 + requirements.txt | 31 ++ scripts/bootstrap.py | 37 ++ scripts/restart.py | 13 + scripts/restart.sh | 4 + scripts/stack.py | 35 ++ scripts/stop-one.py | 12 + scripts/stop-one.sh | 4 + scripts/verify_stack.py | 189 ++++++++ src/__init__.py | 0 src/__main__.py | 9 + src/app.py | 259 +++++++++++ src/bootstrap_storage.py | 42 ++ src/broker.py | 46 ++ src/cleanup.py | 52 +++ src/common.py | 101 ++++ src/dispatcher.py | 44 ++ src/http.py | 95 ++++ src/jobs.py | 54 +++ src/migrate.py | 28 ++ src/offload.py | 16 + src/operator.py | 71 +++ src/render.py | 69 +++ src/storage.py | 74 +++ src/store.py | 174 +++++++ src/worker.py | 210 +++++++++ 41 files changed, 3069 insertions(+), 39 deletions(-) create mode 100644 .gitattributes create mode 100644 ARCHITECTURE.md create mode 100644 Dockerfile create mode 100644 checks/test_render.py create mode 100644 compose.yaml create mode 100644 evidence/integration.json create mode 100644 evidence/restart-processes.json create mode 100644 evidence/stop-one.json create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_jobs.sql create mode 100644 requirements-test.txt create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100644 scripts/restart.py create mode 100644 scripts/restart.sh create mode 100644 scripts/stack.py create mode 100644 scripts/stop-one.py create mode 100644 scripts/stop-one.sh create mode 100644 scripts/verify_stack.py create mode 100644 src/__init__.py create mode 100644 src/__main__.py create mode 100644 src/app.py create mode 100644 src/bootstrap_storage.py create mode 100644 src/broker.py create mode 100644 src/cleanup.py create mode 100644 src/common.py create mode 100644 src/dispatcher.py create mode 100644 src/http.py create mode 100644 src/jobs.py create mode 100644 src/migrate.py create mode 100644 src/offload.py create mode 100644 src/operator.py create mode 100644 src/render.py create mode 100644 src/storage.py create mode 100644 src/store.py create mode 100644 src/worker.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..46526f6a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.png binary +*.jpg binary diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..03610b90 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,15 @@ -## Лабораторная 5 +## ЛР5 — БВТ2301 — Парамонов — на 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Парамонов Глеб, БВТ2301. Целевая оценка5, трек2: идемпотентность versioned pipeline. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Upload202 после durable staging + transactional outbox; Valkey Streams AOF, отдельный worker, три варианта изображения. Lease/fencing, retry/jitter, failed registry/replay, версия2 и очистка staging. Собственная база ЛР4 сохранена с объявленным переходом контракта. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `python scripts/bootstrap.py`, `docker compose up -d --build --wait`; подробности и Python dev setup в [REPORT.md](../REPORT.md). `python -X utf8 scripts/verify_stack.py` воспроизводит persistence/failover, COMMIT/publish gap, три crash points, дубли, retry/replay и reorder v2/v1. Полный прогон PASS; исходные данные в evidence/integration.json. Renderer checks:4 PASS. + +Ограничение: одиночные PG/S3/broker на учебном хосте; не заявляется exactly-once доставка или переносимость измеренных задержек. Источники и использование OpenAI Codex раскрыты в отчёте. + +- [x] Обязательный минимум сохранён с переходом upload201→202. +- [x] Контракт и публичные tests не изменены. +- [x] Offline/OpenAPI/smoke и сценарии отказов выполнены. +- [x] Локальные secrets вне Git. +- [x] README задания сохранён; REPORT описывает реализацию и реальные результаты. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..b0bc2e9c --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,18 @@ +# Durable image pipeline + +```mermaid +flowchart LR + C[Client] --> H[HAProxy] + H --> A[API x2] + A --> S[Private S3 staging] + A --> P[(PostgreSQL attachment + job + outbox)] + P --> D[Dispatcher and lease recovery] + D --> B[(Valkey Streams AOF)] + B --> W[Worker concurrency 1] + W --> R[Bounded renderer subprocess] + W --> F[S3 originals + variants / version] + W --> P + K[Periodic cleanup] --> S +``` + +202 follows durable staging and PostgreSQL COMMIT. Publish can repeat; worker effects use versioned keys and a fenced transaction. ACK follows durable result/retry. Details, limits, reproducible commands and failure evidence: [REPORT.md](REPORT.md). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..558e5171 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +# Google public mirror of the official Docker Python image; immutable digest. +FROM mirror.gcr.io/library/python@sha256:34386ef0cb081344d7ec1c103ba398e6e9f64e9ab3a1509accc92a4e24a07258 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONUTF8=1 \ + APP_HOST=0.0.0.0 APP_PORT=8080 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && \ + groupadd --gid 10001 messenger && useradd --uid 10001 --gid 10001 --no-create-home messenger +COPY src ./src +COPY migrations ./migrations +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=6s --start-period=60s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=5)" +CMD ["python", "-m", "src"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..08baf7cf 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,82 @@ -# Отчёт — лабораторная 5 +# ЛР5 — Парамонов Глеб, БВТ2301 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Целевая оценка **5**, трек 2: доказанная идемпотентность versioned pipeline. Ветка `solution/bvt2301-paramonov`. Авторизация, ACL, курсоры, PostgreSQL, кеш профиля, идемпотентность сообщений, две API-реплики, HAProxy и приватное S3 перенесены из собственной ЛР4. Upload изменён по контракту с 201 ready на 202 queued. Исходные контракты и публичные tests сохранены. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Python 3.12+, Docker Compose v2 (на Windows — Docker Desktop с WSL2). Проверено на Windows 10, i5-11400H, 8GB RAM; WSL2: 4 CPU/3GB RAM/4GB swap. Остальные лабораторные перед запуском остановить: localhost:8080 общий. -## Архитектура и решения +```sh +python -m venv .venv +# Linux/macOS: .venv/bin/python; Windows: .venv/Scripts/python.exe +.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-test.txt +.venv/bin/python scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build --wait +.venv/bin/python scripts/check_contract.py +.venv/bin/python scripts/validate_openapi.py +.venv/bin/python -X utf8 tests/smoke.py +.venv/bin/python -X utf8 scripts/verify_stack.py +docker compose run --rm --no-deps --entrypoint python -v "$PWD/checks:/checks:ro" worker -m unittest discover -s /checks +docker compose down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`make check`, `make validate`, `make test` эквивалентны Python-командам. Интеграционный сценарий занимает несколько минут из-за двух реальных истечений lease по 60 секунд. Он останавливает/восстанавливает только контейнеры текущего Compose project. При прерывании сохраняет `evidence/interrupted-integration.json`; это не PASS. `down` сохраняет named volumes; намеренное удаление только учебных данных этого стенда: `docker compose down -v`. -## Доказательства критериев +Bootstrap один раз генерирует отдельные `.env.api`, `.env.worker`, `.env.storage-admin`, `.env`, `secrets/s3.json`, `secrets/broker.conf`. При повторе сохраняет их; неполный набор не перезаписывает. Секреты игнорируются Git. API имеет Read originals/variants, Write staging; worker отдельные ключи Read/Write/List; admin только bootstrap. URL БД/кеша/S3/брокера задаются соответствующими env, bucket messenger. Только HAProxy опубликован на loopback:8080. HTTPS для локального сценария не заявлен. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Runtime Python 3.12.15, PostgreSQL 17.11, Valkey 8.1, HAProxy 3.0, SeaweedFS 4.48; официальные образы закреплены digest в Dockerfile/compose через mirror.gcr.io (прямой Docker Hub в среде возвращает 403). Все Python-зависимости pinned в requirements.txt, Pillow 12.3.0. Миграции 001–004 применяются автоматически до запуска API/worker. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Устойчивое намерение и обработка -## Эксперименты и отказы +1. API проверяет ACL, MIME и реальное декодирование PNG/JPEG; потоково пишет ограниченный временный файл. До 202 оригинал уже в S3 staging, а attachment, job и outbox записаны одной транзакцией PostgreSQL. Если S3 принял объект, а COMMIT не состоялся, возможен orphan, но подтверждённой клиенту работы нет. +2. Dispatcher читает ограниченные batches outbox под `FOR UPDATE SKIP LOCKED`, публикует в Valkey Streams и затем помечает публикацию. Valkey использует отдельный volume, AOF `appendfsync always`, `noeviction`. Разрыв publish/DB mark даёт допустимый дубль. Queued intent переиздаётся через 30 секунд; БД остаётся источником истины даже при потере broker state. +3. Worker: concurrency=1, prefetch=1; claim под блокировками выдаёт случайный lease_token на 60 секунд и увеличивает attempts. ACK/XDEL выполняются после durable ready/failed/retry. Просроченный processing восстанавливается dispatcher; pending stream entries доступны через XAUTOCLAIM после 65 секунд. +4. Оригинал и три производных записываются под детерминированными versioned keys. Фиксация ready требует совпадения token, версии attachment, processing и неистёкшего lease. Публикация metadata/variants/job done атомарна в PG. Лишь затем staging удаляется; неудачная очистка не откатывает ready. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Инварианты: `(attachment_id,pipeline_version)` уникален; один актуальный lease имеет право опубликовать metadata; терминальный дубль — noop; v1 не пишет в v2 prefix и не меняет текущую v2 metadata. Доставка **at least once**, исполнение PUT может повторяться. Это не exactly-once transport; идемпотентен наблюдаемый результат. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Варианты, ограничения и эксплуатация -## Ограничения и следующие шаги +Оригинал байт-в-байт; EXIF orientation применяется только к производным. Thumbnail JPEG вписывается в 256×256 с floor и без увеличения; crop JPEG 128×128 по центру; optimized WebP вписывается в 1280×1280 без увеличения. Alpha композится на белом, EXIF/ICC и другие metadata удаляются. V1 JPEG85/WebP80, v2 JPEG90/WebP85; subsampling=0, JPEG optimize, WebP method4. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Upload: 10MiB/40Mpx, 2 upload и 1 decoder на реплику, tmpfs32MiB, API512MiB. Decoder worker: дочерний процесс RLIMIT_AS640MiB/CPU20s, wall timeout25s; контейнер768MiB/tmpfs64MiB/1CPU. Глобально максимум1000 queued/processing под advisory lock — сверх лимита явный 503. Broker128MiB/данные64MiB; worker не вытесняет API CPU. Слишком большой по рабочей памяти, но допустимый по размеру файл может завершиться failed/resource_limit: бесконечного повтора нет. + +Максимум5 попыток; transient backoff `min(30,2**attempt)+uniform(0,.5)` секунд. Постоянные decode/checksum/missing/resource ошибки завершаются сразу с безопасным error_code. Реестр failed и управляемый replay: + +```sh +docker compose exec dispatcher python -m src.operator failed +docker compose exec dispatcher python -m src.operator replay JOB_UUID +docker compose exec dispatcher python -m src.operator upgrade ATTACHMENT_UUID 2 +``` + +Replay только failed текущей версии при наличии staging, с audit и reset attempts; done не переисполняется. Upgrade строго повышает поддерживаемую версию и создаёт новое durable intent. Периодический cleanup каждые60s: grace600s, защита активных jobs и известных конечных объектов; failed staging хранится7дней, затем replay без исходника невозможен. Оператор должен разбирать failed до истечения этого срока. Fault injection доступна только локальному оператору через job_faults в PG, не через публичный API. + +## Доказательства + +`scripts/verify_stack.py` создаёт собственные синтетические данные и выполняет реальные kill/restart. Полный первый успешный прогон: `evidence/integration.json`; scopes restart/stop — `restart-processes.json`, `stop-one.json`. ID технические, токены не сохраняются. + +| Критерий / эксперимент | Реальный результат | +| --- | --- | +| Public smoke; session/ACL/async content | PASS, настоящий original + JPEG thumbnail | +| Persistence всех процессов и volumes | PASS, существующая сессия и подтверждённые данные сохранены | +| Failover одной API | 117 successful writes, 2 failed attempts, первое восстановление3.328s; подтверждённое сохранено | +| COMMIT до publish, остановленный worker | 202, queued message201/content409; kill именно принявшей API, restart broker; после старта ready | +| 3 terminal duplicates | metadata неизменна, ровно4 конечных объекта | +| Crash before side effect | exit71, lease_expired, attempt2, ready69.625s, 4 объекта | +| Crash after PUT before DB effect | exit72, lease_expired, attempt2, ready70.219s, 4 объекта | +| Crash after COMMIT before ACK | exit73, ready9.734s, attempt1; дубль noop, 4 объекта | +| 2 transient failures | ready на попытке3 | +| Permanent failure | failed на попытке1, безопасный replay → ready | +| Exhaustion 5 transient failures | failed/retry_exhausted на попытке5, replay → ready | +| Reorder v2 before v1, затем stale v1 | текущая версия2, только4 v2 объекта, metadata не изменилась | +| Размеры/checksums | thumbnail256×160, crop128×128, optimized320×200, SHA/size всех ответов совпали | +| Renderer unit checks | 4 проверки: размеры, alpha/small, EXIF/metadata, центр crop — PASS | + +Гипотеза трека5: transactional intent + versioned keys + fencing сохраняют подтверждённую работу и актуальный результат при дублях, reorder и авариях. Воздействия воспроизводимы одним скриптом. В двух crash cases задержка объясняется lease60s, а не потерей задания; `job_audit` показывает повторный claim. RestartCount Docker может сбрасываться при ручном stop/start, поэтому доказательство точки аварии — записанный перед `os._exit` crash_event и audit, а не разность счётчиков. Это исследование корректности, не throughput benchmark: RPS/p95 CPU для pipeline не заявляются. Времена наблюдения включают polling/API/S3 проверку. + +## Ограничения и источники + +Один хост и одиночные PG/S3/broker; потерю их volumes решение не маскирует. Изоляция памяти ограничивает допустимые изображения сверх простого byte/pixel лимита. Lease не продлевается; слишком медленный эффект отклоняется fencing и повторяется ограниченно. Старые versioned результаты сохранены для известных attachments; автоматическая политика удаления истории версий не реализована. Научный вывод ограничен перечисленными точками отказа, не формальным доказательством всех interleavings. + +Источники: [Valkey persistence](https://valkey.io/topics/persistence/), [XREADGROUP](https://valkey.io/commands/xreadgroup/), [XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [Pillow](https://pillow.readthedocs.io/en/stable/), контракты этого репозитория. Реализация, сценарии, анализ и оформление подготовлены с помощью OpenAI Codex; фактические результаты получены локальными прогонами, помощь ИИ не скрывается. diff --git a/checks/test_render.py b/checks/test_render.py new file mode 100644 index 00000000..d7ab562d --- /dev/null +++ b/checks/test_render.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from PIL import Image +from src.render import render + + +class RenderTests(unittest.TestCase): + def generate(self, image, fmt='PNG', **save_args): + folder=tempfile.TemporaryDirectory(); self.addCleanup(folder.cleanup) + root=Path(folder.name); source=root/'input' + image.save(source,format=fmt,**save_args) + records=render(source,root,1) + return root,{row['name']:row for row in records} + + def test_contract_dimensions(self): + root,rows=self.generate(Image.new('RGB',(320,200),'red')) + self.assertEqual((rows['thumbnail']['width'],rows['thumbnail']['height']),(256,160)) + self.assertEqual((rows['crop']['width'],rows['crop']['height']),(128,128)) + with Image.open(root/'optimized') as optimized: + self.assertEqual(optimized.format,'WEBP') + self.assertEqual(optimized.size,(320,200)) + + def test_alpha_white_and_small_image_not_enlarged(self): + root,rows=self.generate(Image.new('RGBA',(17,9),(255,0,0,0))) + with Image.open(root/'thumbnail') as thumb: + self.assertEqual(thumb.size,(17,9)) + self.assertTrue(all(c>=250 for c in thumb.getpixel((0,0)))) + self.assertEqual(rows['crop']['width'],128) + + def test_exif_orientation_and_metadata_removed(self): + exif=Image.Exif(); exif[274]=6; exif[270]='synthetic metadata must disappear' + root,rows=self.generate(Image.new('RGB',(40,20),'blue'),'JPEG',exif=exif) + with Image.open(root/'thumbnail') as thumb: + self.assertEqual(thumb.size,(20,40)) + self.assertFalse(thumb.getexif()) + + def test_centered_crop(self): + image=Image.new('RGB',(300,100),'red') + image.paste(Image.new('RGB',(100,100),'green'),(100,0)) + root,_=self.generate(image) + with Image.open(root/'crop') as crop: + r,g,b=crop.getpixel((64,64)) + self.assertGreater(g,r+60); self.assertGreater(g,b+60) + + +if __name__=='__main__': unittest.main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..9921cf7b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,142 @@ +x-app: &app + image: mtusi-messenger:lab5 + env_file: .env.api + cpus: 1.0 + mem_limit: 256m + pids_limit: 64 + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 35s + logging: + driver: json-file + options: {max-size: 5m, max-file: "2"} +services: + db: + image: mirror.gcr.io/library/postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/bootstrap.py} + command: [postgres, -c, shared_buffers=64MB, -c, max_connections=30] + volumes: [pgdata:/var/lib/postgresql/data] + cpus: 1.0 + mem_limit: 256m + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + cache: + image: mirror.gcr.io/valkey/valkey@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: [valkey-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + cpus: 0.25 + mem_limit: 64m + healthcheck: + test: [CMD, valkey-cli, ping] + interval: 2s + timeout: 1s + retries: 20 + migrate: + <<: *app + command: [python, -m, src.migrate] + healthcheck: {disable: true} + depends_on: + db: {condition: service_healthy} + api: + <<: *app + build: . + deploy: + replicas: 2 + cpus: 0.5 + mem_limit: 512m + tmpfs: ["/tmp:size=32m,mode=1777"] + depends_on: + migrate: {condition: service_completed_successfully} + cache: {condition: service_healthy} + storage-init: {condition: service_completed_successfully} + s3: + image: mirror.gcr.io/chrislusf/seaweedfs@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d + user: "1000:1000" + command: [server, -dir=/data, -ip=127.0.0.1, -ip.bind=127.0.0.1, -s3, -s3.ip.bind=0.0.0.0, -s3.config=/etc/seaweedfs/s3.json, -s3.port.iceberg=0, -s3.port.lance=0, -s3.autoCreateBucket=false, -volume.max=10, -master.volumeSizeLimitMB=100, -filer.concurrentUploadLimitMB=64] + environment: {GOMEMLIMIT: 384MiB, GOMAXPROCS: "2"} + volumes: ["s3data:/data", "./secrets/s3.json:/etc/seaweedfs/s3.json:ro"] + cpus: 1.0 + mem_limit: 512m + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 20s + storage-init: + <<: *app + env_file: .env.storage-admin + command: [python, -m, src.bootstrap_storage] + healthcheck: {disable: true} + depends_on: [s3] + reconciler: + <<: *app + env_file: .env.worker + cpus: 0.25 + mem_limit: 192m + command: [python, -m, src.cleanup] + healthcheck: + test: [CMD, python, -c, "import os; os.kill(1,0)"] + interval: 10s + timeout: 2s + start_period: 5s + retries: 3 + depends_on: + migrate: {condition: service_completed_successfully} + storage-init: {condition: service_completed_successfully} + broker: + image: mirror.gcr.io/valkey/valkey@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: [valkey-server, /etc/valkey/lab.conf] + volumes: ["brokerdata:/data", "./secrets/broker.conf:/etc/valkey/lab.conf:ro"] + cpus: 0.25 + mem_limit: 128m + dispatcher: + <<: *app + env_file: .env.worker + cpus: 0.25 + mem_limit: 128m + tmpfs: ["/tmp:size=32m,mode=1777"] + command: [python, -m, src.dispatcher] + healthcheck: + test: [CMD, python, -c, "import os; os.kill(1,0)"] + interval: 10s + timeout: 2s + start_period: 5s + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_started} + worker: + <<: *app + env_file: .env.worker + cpus: 1.0 + mem_limit: 768m + tmpfs: ["/tmp:size=64m,mode=1777"] + restart: on-failure:5 + command: [python, -m, src.worker] + healthcheck: + test: [CMD, python, -c, "import os; os.kill(1,0)"] + interval: 10s + timeout: 2s + start_period: 5s + depends_on: + migrate: {condition: service_completed_successfully} + storage-init: {condition: service_completed_successfully} + broker: {condition: service_started} + lb: + image: mirror.gcr.io/library/haproxy@sha256:d84eb36e91830bcaa67b23cfde1909850b99f11eb60da3abd185bd0daf3f75bb + ports: ["127.0.0.1:8080:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + cpus: 0.5 + mem_limit: 64m + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + depends_on: + api: {condition: service_healthy} +volumes: + pgdata: + s3data: + brokerdata: diff --git a/evidence/integration.json b/evidence/integration.json new file mode 100644 index 00000000..68888909 --- /dev/null +++ b/evidence/integration.json @@ -0,0 +1,177 @@ +{ + "public_smoke": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.", + "commit_before_publish": { + "attachment_id": "66920d96-7970-4fe2-a3be-bf609ba61723", + "job_id": "ccfee7d8-f34d-4f94-96cb-c070ae3c814c", + "killed_api": "2d379305b85819558b18822a5445a8498e01e262415646bc786a969b7556e947", + "broker_restarted": true, + "queued_message_accepted": true, + "original_and_variants_verified": true + }, + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 117, \"failed_attempts\": 2, \"first_recovery_seconds\": 3.328}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "terminal_duplicates": { + "deliveries": 3, + "effect_objects": 4, + "metadata_unchanged": true + }, + "crashes": [ + { + "point": "before_effect", + "seconds_to_ready": 69.625, + "attempts": 2, + "effect_objects": 4, + "audit": [ + { + "event": "claimed", + "details": { + "attempt": 1, + "version": 1 + } + }, + { + "event": "lease_expired", + "details": { + "attempt": 1, + "outcome": "queued" + } + }, + { + "event": "claimed", + "details": { + "attempt": 2, + "version": 1 + } + }, + { + "event": "ready", + "details": { + "attempt": 2, + "version": 1 + } + } + ], + "crash_event": { + "event": "injected_crash", + "point": "before_effect", + "job_id": "ca4b66aa-34a1-4768-9bce-c18b46c31726", + "exit_code": 71 + }, + "restart_count_before": 0, + "restart_count_after": 1 + }, + { + "point": "after_effect", + "seconds_to_ready": 70.219, + "attempts": 2, + "effect_objects": 4, + "audit": [ + { + "event": "claimed", + "details": { + "attempt": 1, + "version": 1 + } + }, + { + "event": "lease_expired", + "details": { + "attempt": 1, + "outcome": "queued" + } + }, + { + "event": "claimed", + "details": { + "attempt": 2, + "version": 1 + } + }, + { + "event": "ready", + "details": { + "attempt": 2, + "version": 1 + } + } + ], + "crash_event": { + "event": "injected_crash", + "point": "after_effect", + "job_id": "a3f1c5c3-365e-4193-af3b-7af2a6a1fb61", + "exit_code": 72 + }, + "restart_count_before": 1, + "restart_count_after": 1 + }, + { + "point": "after_commit", + "seconds_to_ready": 9.734, + "attempts": 1, + "effect_objects": 4, + "audit": [ + { + "event": "claimed", + "details": { + "attempt": 1, + "version": 1 + } + }, + { + "event": "ready", + "details": { + "attempt": 1, + "version": 1 + } + } + ], + "crash_event": { + "event": "injected_crash", + "point": "after_commit", + "job_id": "139ad7ce-0fde-4726-9ce0-d3d1366af2e3", + "exit_code": 73 + }, + "restart_count_before": 1, + "restart_count_after": 1 + } + ], + "retries": [ + { + "fault": "transient", + "injected": 2, + "terminal_status": "ready", + "error_code": null, + "attempts": 3, + "replay_ready": false + }, + { + "fault": "permanent", + "injected": 1, + "terminal_status": "failed", + "error_code": "synthetic_permanent", + "attempts": 1, + "replay_ready": true + }, + { + "fault": "transient", + "injected": 5, + "terminal_status": "failed", + "error_code": "retry_exhausted", + "attempts": 5, + "replay_ready": true + } + ], + "version_reorder": { + "old_job_id": "d54519c7-ec5f-4a89-b399-a56c22d1b2c8", + "new_job_id": "03b47d0b-e692-4042-8c3e-4f4c5a58ce2e", + "final_pipeline_version": 2, + "keys": [ + "originals/9931ab06-7f0c-4f8d-8759-ba08d336ee83/v2", + "variants/9931ab06-7f0c-4f8d-8759-ba08d336ee83/v2/crop", + "variants/9931ab06-7f0c-4f8d-8759-ba08d336ee83/v2/optimized", + "variants/9931ab06-7f0c-4f8d-8759-ba08d336ee83/v2/thumbnail" + ], + "stale_delivery_did_not_change_result": true + }, + "passed": true +} diff --git a/evidence/restart-processes.json b/evidence/restart-processes.json new file mode 100644 index 00000000..fc020dc2 --- /dev/null +++ b/evidence/restart-processes.json @@ -0,0 +1,759 @@ +{ + "before": { + "e207ed1f45a99ac436101bcf1939d5849db4bcf12ec04a381350ca5843eaee8d": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 15900, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:15:48.648730118Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.941228427Z", + "End": "2026-10-16T19:15:54.491000512Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:15:59.491668372Z", + "End": "2026-10-16T19:16:00.020617257Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:06.209442491Z", + "End": "2026-10-16T19:16:06.692932737Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:11.693747243Z", + "End": "2026-10-16T19:16:12.261470207Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "2d379305b85819558b18822a5445a8498e01e262415646bc786a969b7556e947": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 16836, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:03.778477014Z", + "FinishedAt": "2026-10-16T19:16:01.778153812Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:54.159744102Z", + "End": "2026-10-16T19:15:54.691565483Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:15:59.692451426Z", + "End": "2026-10-16T19:16:00.272887194Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:09.438288986Z", + "End": "2026-10-16T19:16:09.987700267Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:14.989013652Z", + "End": "2026-10-16T19:16:15.48131061Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "30388d61f005c42478ebc7a9b2e474526e5679653ad02d71062f570a33339b42": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 99476, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:09:22.387752373Z", + "FinishedAt": "2026-10-16T19:09:21.196406142Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:16:08.63406217Z", + "End": "2026-10-16T19:16:08.673149107Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:10.67433871Z", + "End": "2026-10-16T19:16:10.74446595Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:12.74580529Z", + "End": "2026-10-16T19:16:12.819158406Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:14.819725559Z", + "End": "2026-10-16T19:16:14.860719841Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:16.862062759Z", + "End": "2026-10-16T19:16:16.901318583Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "2482c50ba04375457b8b6422d2f6f9db8ded4e28e92c8a9a8c0e046884a83243": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 99355, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:09:21.764314954Z", + "FinishedAt": "2026-10-16T19:09:21.101670648Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:16:08.062041841Z", + "End": "2026-10-16T19:16:08.081795218Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:10.082496788Z", + "End": "2026-10-16T19:16:10.101096322Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:12.102152091Z", + "End": "2026-10-16T19:16:12.117714807Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:14.119003078Z", + "End": "2026-10-16T19:16:14.136254833Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:16.136981435Z", + "End": "2026-10-16T19:16:16.161114332Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "170d93adff3b565196bb1c836d059f0e7290c37ee5505664a28bea51c3b2e28b": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 16288, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:15:54.91848616Z", + "FinishedAt": "2026-10-16T19:15:40.094745008Z" + } + }, + "4a3eb650e55ebdd3b336789bc2104a362c5d2852674a6cef16b61cd989977fc3": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 863, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:09:41.38119697Z", + "FinishedAt": "2026-10-16T19:09:41.100972388Z" + } + }, + "25371d1548f3a0d654c4b0bdf1e67bc7bbbdf64432a5976dfebf35b216f0db10": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 15884, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:15:48.636642859Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.805713572Z", + "End": "2026-10-16T19:15:53.840774275Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:04.489169743Z", + "End": "2026-10-16T19:16:04.517075806Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:15.058571127Z", + "End": "2026-10-16T19:16:15.106536039Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "dca968acf9fa185775265fd5c02f3168f900ae43d1a80107981af3c2c3e5b648": { + "service": "broker", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 16750, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:03.267282441Z", + "FinishedAt": "2026-10-16T19:16:02.306854751Z" + } + }, + "836d21386914945332563a9a187d8f9303bad48cd17b3f5ed41a49b8dbac1794": { + "service": "worker", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 17349, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:12.595988209Z", + "FinishedAt": "2026-10-16T19:15:59.472133332Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.925132605Z", + "End": "2026-10-16T19:15:53.957650902Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:17.702002693Z", + "End": "2026-10-16T19:16:17.738665652Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "a3100732982bf1a414c197e1258e2d5899a12c56d6acdaf00cce22d883d782db": { + "service": "dispatcher", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 17371, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:12.607708694Z", + "FinishedAt": "2026-10-16T19:15:59.486298003Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.925223357Z", + "End": "2026-10-16T19:15:53.961272011Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:17.761882647Z", + "End": "2026-10-16T19:16:17.802075226Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + } + }, + "after": { + "e207ed1f45a99ac436101bcf1939d5849db4bcf12ec04a381350ca5843eaee8d": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18213, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:21.245597195Z", + "FinishedAt": "2026-10-16T19:16:20.35897279Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:16:11.693747243Z", + "End": "2026-10-16T19:16:12.261470207Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:17.262275872Z", + "End": "2026-10-16T19:16:17.87718833Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:26.951395292Z", + "End": "2026-10-16T19:16:27.477696698Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:16:32.478305016Z", + "End": "2026-10-16T19:16:32.976822728Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:16:39.682312094Z", + "End": "2026-10-16T19:16:40.181239899Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "2d379305b85819558b18822a5445a8498e01e262415646bc786a969b7556e947": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18308, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:21.562721601Z", + "FinishedAt": "2026-10-16T19:16:20.460269785Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:16:09.438288986Z", + "End": "2026-10-16T19:16:09.987700267Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:14.989013652Z", + "End": "2026-10-16T19:16:15.48131061Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:26.967414962Z", + "End": "2026-10-16T19:16:27.491810666Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:16:32.492721407Z", + "End": "2026-10-16T19:16:32.98580126Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:16:39.691161743Z", + "End": "2026-10-16T19:16:40.181217764Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "30388d61f005c42478ebc7a9b2e474526e5679653ad02d71062f570a33339b42": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 17939, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:20.036920618Z", + "FinishedAt": "2026-10-16T19:16:19.439332406Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:16:31.816469712Z", + "End": "2026-10-16T19:16:31.858783923Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:34.712013593Z", + "End": "2026-10-16T19:16:34.752450209Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:37.605331208Z", + "End": "2026-10-16T19:16:37.653046123Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:39.654175895Z", + "End": "2026-10-16T19:16:39.694929525Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:16:41.696275853Z", + "End": "2026-10-16T19:16:41.772118978Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "2482c50ba04375457b8b6422d2f6f9db8ded4e28e92c8a9a8c0e046884a83243": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18140, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:21.023413285Z", + "FinishedAt": "2026-10-16T19:16:19.513572429Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:16:31.877806845Z", + "End": "2026-10-16T19:16:31.894884333Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:34.748325592Z", + "End": "2026-10-16T19:16:34.766419492Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:37.619565901Z", + "End": "2026-10-16T19:16:37.649277011Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:39.650210957Z", + "End": "2026-10-16T19:16:39.668504015Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:16:41.66923856Z", + "End": "2026-10-16T19:16:41.718053345Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "170d93adff3b565196bb1c836d059f0e7290c37ee5505664a28bea51c3b2e28b": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18099, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:20.875893593Z", + "FinishedAt": "2026-10-16T19:16:19.341244767Z" + } + }, + "4a3eb650e55ebdd3b336789bc2104a362c5d2852674a6cef16b61cd989977fc3": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 19105, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:41.326090262Z", + "FinishedAt": "2026-10-16T19:16:41.073794175Z" + } + }, + "25371d1548f3a0d654c4b0bdf1e67bc7bbbdf64432a5976dfebf35b216f0db10": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18247, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:21.385006702Z", + "FinishedAt": "2026-10-16T19:16:20.364051992Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.805713572Z", + "End": "2026-10-16T19:15:53.840774275Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:04.489169743Z", + "End": "2026-10-16T19:16:04.517075806Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:15.058571127Z", + "End": "2026-10-16T19:16:15.106536039Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:26.977241704Z", + "End": "2026-10-16T19:16:27.035513863Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:38.741083561Z", + "End": "2026-10-16T19:16:38.771444984Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "dca968acf9fa185775265fd5c02f3168f900ae43d1a80107981af3c2c3e5b648": { + "service": "broker", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18031, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:20.389002252Z", + "FinishedAt": "2026-10-16T19:16:19.42199244Z" + } + }, + "836d21386914945332563a9a187d8f9303bad48cd17b3f5ed41a49b8dbac1794": { + "service": "worker", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18094, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:20.683352103Z", + "FinishedAt": "2026-10-16T19:16:19.669683373Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.925132605Z", + "End": "2026-10-16T19:15:53.957650902Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:17.702002693Z", + "End": "2026-10-16T19:16:17.738665652Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:26.706971267Z", + "End": "2026-10-16T19:16:26.737171807Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:38.442491315Z", + "End": "2026-10-16T19:16:38.477291779Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "a3100732982bf1a414c197e1258e2d5899a12c56d6acdaf00cce22d883d782db": { + "service": "dispatcher", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 18175, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:16:21.132738836Z", + "FinishedAt": "2026-10-16T19:16:19.97523507Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:15:53.925223357Z", + "End": "2026-10-16T19:15:53.961272011Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:17.761882647Z", + "End": "2026-10-16T19:16:17.802075226Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:26.840557382Z", + "End": "2026-10-16T19:16:26.869490976Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:16:38.575762376Z", + "End": "2026-10-16T19:16:38.608636915Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + } + }, + "volumes_removed": false +} diff --git a/evidence/stop-one.json b/evidence/stop-one.json new file mode 100644 index 00000000..e5ae0f9a --- /dev/null +++ b/evidence/stop-one.json @@ -0,0 +1,11 @@ +{ + "target_instance": "6ea29cdfce3d46d287ca07f0a69b571e", + "stopped_container": "e207ed1f45a99ac436101bcf1939d5849db4bcf12ec04a381350ca5843eaee8d", + "before_containers": [ + "e207ed1f45a99ac436101bcf1939d5849db4bcf12ec04a381350ca5843eaee8d", + "2d379305b85819558b18822a5445a8498e01e262415646bc786a969b7556e947" + ], + "after_running_containers": [ + "2d379305b85819558b18822a5445a8498e01e262415646bc786a969b7556e947" + ] +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..48689f30 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,30 @@ +global + log stdout format raw local0 + maxconn 256 +defaults + mode http + log global + timeout connect 1s + timeout client 35s + timeout server 32s + timeout check 1s + timeout queue 2s + retries 0 + option http-server-close +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s +frontend public + bind :8080 + default_backend api +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template replica 1-4 api:8080 check inter 500ms fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr none maxconn 32 diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..3b9cda97 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,35 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX sessions_expiry ON sessions(expires_at); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY(chat_id,user_id) +); +CREATE INDEX members_user_chat ON members(user_id,chat_id); +CREATE INDEX chats_order ON chats(created_at,id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY(chat_id,sender_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX messages_history ON messages(chat_id,created_at,id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..582b0c4f --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY(actor_id,chat_id,key) +); +CREATE INDEX idempotency_expiry ON idempotency(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..839d6753 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,25 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + storage_state text NOT NULL DEFAULT 'ready' CHECK (storage_state IN ('ready','missing')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id), + FOREIGN KEY (chat_id, owner_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX attachments_chat ON attachments(chat_id,created_at,id); +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_content_check + CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0)); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + PRIMARY KEY(message_id,attachment_id) +); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 00000000..23d39c72 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' + CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version>0); +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN accepted_at timestamptz NOT NULL DEFAULT clock_timestamp(); + +CREATE TABLE jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK (pipeline_version IN (1,2)), + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','done','failed','superseded')), + attempts integer NOT NULL DEFAULT 0 CHECK (attempts>=0), + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + error_code text, + traceparent text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (attachment_id,pipeline_version) +); +CREATE INDEX jobs_due ON jobs(status,next_attempt_at); +CREATE TABLE outbox ( + job_id uuid PRIMARY KEY REFERENCES jobs(id), + published_at timestamptz, + publications integer NOT NULL DEFAULT 0 +); +CREATE TABLE job_audit ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs(id), + event text NOT NULL, + details jsonb NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE job_faults ( + job_id uuid PRIMARY KEY REFERENCES jobs(id), + point text NOT NULL CHECK (point IN ('before_effect','after_effect','after_commit','transient','permanent')), + remaining integer NOT NULL CHECK (remaining>=0) +); diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..9dff8880 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +-r requirements.txt +httpx==0.28.1 +matplotlib==3.10.8 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..cc889419 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +fastapi==0.143.2 +h11==0.16.0 +idna==3.20 +opentelemetry-api==1.45.1 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.11 +pydantic==2.14.1 +pydantic_core==2.50.1 +redis==8.1.0 +starlette==1.8.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +tzdata==2026.5 +uvicorn==0.54.0 +boto3==1.43.114 +Pillow==12.3.0 +botocore==1.43.114 +jmespath==1.1.0 +s3transfer==0.19.2 +python-dateutil==2.9.0.post0 +urllib3==2.8.0 +six==1.17.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..2caaa3be --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,37 @@ +"""Create local, separate API/worker/storage-admin credentials; never print values.""" +import json +from pathlib import Path +import secrets + +root=Path(__file__).resolve().parents[1] +required=['.env.api','.env.worker','.env.storage-admin','secrets/s3.json','secrets/broker.conf'] +if (root/'.env').exists(): + if not all((root/p).exists() for p in required): + raise SystemExit('Incomplete local configuration; preserve and restore missing files') + print('Existing local configuration preserved') +else: + password=secrets.token_hex(24) + broker_password=secrets.token_hex(24) + keys={role:(secrets.token_hex(12),secrets.token_hex(32)) for role in ('admin','api','worker')} + common=f'DATABASE_URL=postgresql://messenger:{password}@db:5432/messenger\nREDIS_URL=redis://cache:6379/0\nCURSOR_SECRET={secrets.token_hex(32)}\nSESSION_TTL_SECONDS=3600\n' + def storage(role): + key,secret=keys[role] + return f'S3_ENDPOINT=http://s3:8333\nS3_BUCKET=messenger\nS3_ACCESS_KEY={key}\nS3_SECRET_KEY={secret}\n' + files={'.env':f'POSTGRES_PASSWORD={password}\n', + '.env.api':common+storage('api'), + '.env.worker':common+storage('worker')+f'BROKER_URL=redis://:{broker_password}@broker:6379/0\n', + '.env.storage-admin':storage('admin'), + 'secrets/broker.conf':f'bind 0.0.0.0\nprotected-mode yes\nrequirepass {broker_password}\nappendonly yes\nappendfsync always\ndir /data\nsave ""\nmaxmemory 64mb\nmaxmemory-policy noeviction\nauto-aof-rewrite-min-size 16mb\n', + 'secrets/s3.json':json.dumps({'identities':[ + {'name':role,'credentials':[{'accessKey':keys[role][0],'secretKey':keys[role][1]}], 'actions':actions} + for role,actions in [ + ('admin',['Admin','Read','Write','List','Tagging']), + ('api',['Read:messenger/originals/*','Read:messenger/variants/*','Write:messenger/staging/*']), + ('worker',['Read:messenger/*','Write:messenger/originals/*','Write:messenger/variants/*','Write:messenger/staging/*','List:messenger'])] + ]},indent=2)+'\n'} + (root/'secrets').mkdir(exist_ok=True) + for name,content in files.items(): + with (root/name).open('x',encoding='utf-8') as file: file.write(content) + try: (root/name).chmod(0o600) + except OSError: pass + print('Created separate local API, worker and admin configuration; values not displayed') diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..0e923c95 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,13 @@ +"""Restart every running replica and dependency, preserving durable volumes.""" +import json +from stack import ROOT, docker, state + +before = state() +docker('compose', 'restart', 'api', 'db', 'cache', 'lb', 's3', 'reconciler', 'broker', 'worker', 'dispatcher') +after = state() +assert set(before) == set(after) +assert all(before[c]['state']['StartedAt'] != after[c]['state']['StartedAt'] for c in before) +(ROOT / 'evidence').mkdir(exist_ok=True) +(ROOT / 'evidence/restart-processes.json').write_text( + json.dumps({'before': before, 'after': after, 'volumes_removed': False}, indent=2) + '\n', + encoding='utf-8') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..d504e1e1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/restart.py diff --git a/scripts/stack.py b/scripts/stack.py new file mode 100644 index 00000000..33a4cb73 --- /dev/null +++ b/scripts/stack.py @@ -0,0 +1,35 @@ +"""Shared bounded Docker helpers for reproducible, scoped failure experiments.""" +import json +from pathlib import Path +import subprocess +import time +ROOT=Path(__file__).resolve().parents[1] +def docker(*args): + return subprocess.check_output(['docker',*args],cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip() +def replicas(): return docker('compose','ps','-q','api').splitlines() + +def ready_replicas(count=2,timeout=60): + deadline=time.monotonic()+timeout + while time.monotonic()10: invalid('Invalid attachments') + ids=[uuid_value(v) for v in ids] + if len(ids)!=len(set(ids)): invalid('Duplicate attachment') + text=string(body['text'],4000,blank=True) + if not text.strip() and not ids: invalid('Message must have content') + return await store.create_message(user['id'],chat_id,text,key,ids) + + def metadata(row): + fields=('id','chat_id','owner_id','filename','content_type','size_bytes','sha256','created_at','updated_at','status','error_code') + result={k:row[k] for k in fields} + result['variants']=[{k:v for k,v in variant.items() if k!='object_key'} for variant in row['variants']] if row['status']=='ready' else [] + return result + + @api.post('/api/v1/chats/{chat_id}/attachments',status_code=202) + async def upload(chat_id:str,request:Request): + user=await actor(request); chat_id=uuid_value(chat_id) + await store.chat(user['id'],chat_id) + if set(request.query_params)!={'filename'} or len(request.query_params.getlist('filename'))!=1: + invalid('Expected one filename') + filename=string(request.query_params['filename'],255) + if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in filename): invalid('Invalid filename') + mime=request.headers.get('content-type','').strip().lower() + if mime not in ('image/png','image/jpeg'): raise ApiError(415,'unsupported_media_type','Expected PNG/JPEG') + if uploads.locked(): raise ApiError(503,'unavailable','Upload slots busy',{'Retry-After':'1'}) + async with uploads: + with tempfile.TemporaryFile() as file: + size=0 + async for chunk in request.stream(): + size+=len(chunk) + if size>MAX_BYTES: raise ApiError(413,'payload_too_large','Upload exceeds 10 MiB') + file.write(chunk) + async with decoders: + digest=await complete_thread(validate_image,file,mime) + row=await store.save_attachment(user['id'],chat_id,filename,mime,size,digest,file,storage) + return metadata(row) + + @api.get('/api/v1/attachments/{attachment_id}') + async def attachment(attachment_id:str,request:Request): + user=await actor(request) + return metadata(await store.attachment(user['id'],uuid_value(attachment_id))) + + @api.get('/api/v1/attachments/{attachment_id}/content') + async def content(attachment_id:str,request:Request): + user=await actor(request) + row=await store.attachment(user['id'],uuid_value(attachment_id)) + if row['status']!='ready': raise ApiError(409,'conflict','Attachment is not ready') + if set(request.query_params)-{'variant'} or len(request.query_params.getlist('variant'))>1: invalid('Invalid query') + variant=request.query_params.get('variant','original') + target=row if variant=='original' else next((v for v in row['variants'] if v['name']==variant),None) + if target is None: raise ApiError(404,'not_found','Variant not found') + if downloads.locked(): raise ApiError(503,'unavailable','Download slots busy') + await downloads.acquire() + try: + obj=await asyncio.to_thread(storage.get,target['object_key']) + except BaseException: + downloads.release() + raise + async def chunks(): + try: + while data:=await asyncio.to_thread(obj['Body'].read,65536): + yield data + finally: + obj['Body'].close() + downloads.release() + return StreamingResponse(chunks(),media_type=target['content_type'],headers={ + 'Content-Length':str(target['size_bytes']),'X-Content-Type-Options':'nosniff', + 'Content-Disposition':'attachment'}) + + api.add_middleware(Boundaries,settings=cfg) + return api + + +app=create_app() diff --git a/src/bootstrap_storage.py b/src/bootstrap_storage.py new file mode 100644 index 00000000..574a1d20 --- /dev/null +++ b/src/bootstrap_storage.py @@ -0,0 +1,42 @@ +"""Idempotent private bucket bootstrap; only this one-shot service receives admin credentials.""" +import time +from botocore.exceptions import BotoCoreError, ClientError +from .storage import Storage, HEALTH_KEY, HEALTH_BYTES + +storage=Storage() +for attempt in range(60): + try: + storage.client.list_buckets() + break + except (BotoCoreError,ClientError): + if attempt==59: raise RuntimeError('Object storage did not become available') from None + time.sleep(1) +try: + storage.client.head_bucket(Bucket=storage.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('404','NoSuchBucket'): raise + storage.client.create_bucket(Bucket=storage.bucket) +# No anonymous identity and no public bucket policy. Do not overwrite any policy +# silently on repeat; reject a public policy if an operator added one accidentally. +try: + policy=storage.client.get_bucket_policy(Bucket=storage.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('NoSuchBucketPolicy','404'): raise +else: + raise RuntimeError('Unexpected bucket policy; inspect it before starting API') +deadline=time.monotonic()+90 +while True: + try: + try: + storage.client.head_object(Bucket=storage.bucket, Key=HEALTH_KEY) + except ClientError as exc: + if exc.response['Error']['Code'] not in ('404','NoSuchKey'): raise + # Fixed, reserved key/content: retry after uncertain PUT is safe. + storage.client.put_object(Bucket=storage.bucket, Key=HEALTH_KEY, Body=HEALTH_BYTES) + storage.ready() + break + except (BotoCoreError,ClientError): + if time.monotonic()>=deadline: + raise RuntimeError('Storage volumes did not become ready within bootstrap deadline') from None + time.sleep(1) +print('Private bucket ready; bootstrap is repeatable') diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..5fe09c07 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,46 @@ +"""Persistent Valkey Streams transport; database outbox remains the durable intent.""" +import json +import os + +import redis + +STREAM = 'attachment-jobs' +GROUP = 'image-workers' + + +def connect(): + return redis.Redis.from_url(os.environ['BROKER_URL'], decode_responses=True, + socket_connect_timeout=2, socket_timeout=3) + + +def ensure_group(client): + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except redis.ResponseError as exc: + if not str(exc).startswith('BUSYGROUP'): + raise + + +def publish(client, payload): + # appendfsync always in Compose: successful XADD is the broker acceptance + # boundary. This is NOT an atomic transaction with PostgreSQL's outbox update. + return client.xadd(STREAM, {'job': json.dumps(payload, separators=(',', ':'))}) + + +def receive(client, consumer): + ensure_group(client) + claimed = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=65_000, + start_id='0-0', count=1) + if claimed[1]: + return claimed[1][0] + items = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000) + return items[0][1][0] if items else None + + +def acknowledge(client, message_id): + # Called only after durable terminal/no-op outcome (or durable scheduled retry). + # One consumer group owns this stream. Ack+delete prevents unbounded history. + with client.pipeline(transaction=True) as pipe: + pipe.xack(STREAM, GROUP, message_id) + pipe.xdel(STREAM, message_id) + pipe.execute() diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 00000000..f79c2834 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,52 @@ +"""Version-aware staging retention. Never deletes active jobs or referenced final effects.""" +from datetime import datetime, timezone +import json +import signal +import threading + +from .jobs import database +from .storage import HEALTH_KEY, Storage + +stop=threading.Event() + + +def run(): + storage=Storage(); deleted=0; scanned=0 + try: + with database() as db: + for prefix in ('staging/','originals/','variants/'): + for page in storage.client.get_paginator('list_objects_v2').paginate(Bucket=storage.bucket,Prefix=prefix): + for obj in page.get('Contents',[]): + key=obj['Key']; scanned+=1 + if key==HEALTH_KEY: continue + age=datetime.now(timezone.utc).timestamp()-obj['LastModified'].timestamp() + if age<600: continue + with db.transaction(): + if not db.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS ok',(key,)).fetchone()['ok']: continue + if prefix=='staging/': + att=db.execute('SELECT * FROM attachments WHERE staging_key=%s FOR UPDATE',(key,)).fetchone() + if att and (att['status'] in ('queued','processing') or (att['status']=='failed' and age<7*86400)): + continue + # Ready staging is obsolete. Failed staging retained 7 days; + # explicit operator replay afterwards requires a restored source. + storage.delete(key); deleted+=1 + if att: + db.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s',(att['id'],key)) + else: + # Keep every version for a known attachment; old versions + # are bounded by the two supported pipelines and aid audit. + part=key.split('/')[1] + row=db.execute('SELECT 1 FROM attachments WHERE id::text=%s',(part,)).fetchone() + if row: continue + storage.delete(key); deleted+=1 + return {'scanned':scanned,'deleted':deleted} + finally: storage.client.close() + + +if __name__=='__main__': + signal.signal(signal.SIGTERM,lambda *_:stop.set()) + signal.signal(signal.SIGINT,lambda *_:stop.set()) + while not stop.is_set(): + try: print(json.dumps({'event':'cleanup',**run()}),flush=True) + except Exception as exc: print(json.dumps({'event':'cleanup_retry','type':type(exc).__name__}),flush=True) + stop.wait(60) diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..7d4e9e84 --- /dev/null +++ b/src/common.py @@ -0,0 +1,101 @@ +"""Strict contract validation and resource-bound cursors.""" +import base64 +import binascii +import hashlib +import hmac +import json +import re +from datetime import datetime, timezone +from uuid import UUID + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status, self.code, self.message = status, code, message + self.headers = headers or {} + + +def invalid(message='Invalid request'): + raise ApiError(400, 'invalid_request', message) + + +def uuid_value(value): + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}', value + ): + invalid('Invalid UUID') + return str(UUID(value)) + + +def string(value, maximum, *, blank=False): + if not isinstance(value, str) or len(value) > maximum or (not blank and not value.strip()): + invalid('Invalid string') + # JSON can encode unpaired surrogates; they are not Unicode scalar values. + try: + value.encode('utf-8') + except UnicodeError: + invalid('Invalid Unicode') + return value + + +def object_fields(value, required, optional=()): + if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): + invalid('Invalid fields') + return value + + +def timestamp(): + return datetime.now(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def encode_cursor(resource, key, secret): + data = json.dumps([resource, *key], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(data + hmac.digest(secret, data, 'sha256')).decode().rstrip('=') + + +def decode_cursor(value, resource, secret): + try: + if len(value) > 1024 or not re.fullmatch(r'[A-Za-z0-9_-]+', value): + raise ValueError() + raw = base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True) + data, signature = raw[:-32], raw[-32:] + if not hmac.compare_digest(hmac.digest(secret, data, 'sha256'), signature): + raise ValueError() + scope, created, identifier = json.loads(data) + if scope != resource or not isinstance(created, str) or not created.endswith('Z'): + raise ValueError() + datetime.fromisoformat(created.replace('Z', '+00:00')) + UUID(identifier) + return created, identifier + except (ValueError, TypeError, binascii.Error, UnicodeError): + invalid('Invalid cursor') + + +def pagination(request, resource, secret): + if any(len(request.query_params.getlist(k)) != 1 for k in request.query_params): + invalid('Duplicate query parameter') + if set(request.query_params) - {'limit', 'cursor'}: + invalid('Unknown query parameter') + value = request.query_params.get('limit', '50') + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + invalid('Limit must be 1..100') + cursor = request.query_params.get('cursor') + return int(value), decode_cursor(cursor, resource, secret) if cursor is not None else None + + +async def json_body(request): + if request.headers.get('content-type', '').split(';')[0].strip().lower() != 'application/json': + raise ApiError(415, 'unsupported_media_type', 'Expected application/json') + def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate key') + result[key] = value + return result + def no_constant(value): + raise ValueError('non-finite number') + try: + return json.loads((await request.body()).decode('utf-8'), object_pairs_hook=unique_pairs, parse_constant=no_constant) + except (ValueError, UnicodeError, RecursionError): + invalid('Invalid JSON') diff --git a/src/dispatcher.py b/src/dispatcher.py new file mode 100644 index 00000000..65dc546d --- /dev/null +++ b/src/dispatcher.py @@ -0,0 +1,44 @@ +"""Recover committed intent automatically and republish after uncertain acceptance.""" +import json +import signal +import threading + +from .broker import connect, ensure_group, publish +from .jobs import database, payload, recover_expired + +stop = threading.Event() + + +def tick(): + with database() as db, connect() as client: + recover_expired(db) + ensure_group(client) + with db.transaction(): + # Bounded batch and SKIP LOCKED allow another dispatcher without + # duplicate simultaneous publication. Crash after XADD may redeliver. + rows = db.execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id + WHERE j.status='queued' AND j.next_attempt_at<=clock_timestamp() + AND (o.published_at IS NULL OR o.published_at= self.cfg.max_inflight: + raise ApiError(503, 'unavailable', 'Too many in-flight requests', {'Retry-After': '1'}) + now = time.monotonic() + self.tokens = min(self.cfg.write_burst, self.tokens + (now - self.updated) * self.cfg.write_rps) + self.updated = now + if scope['method'] in ('POST', 'PUT', 'PATCH'): + if self.tokens < 1: + raise ApiError(429, 'rate_limited', 'Write rate exceeded', {'Retry-After': '1'}) + self.tokens -= 1 + self.active += 1 + admitted = True + # Binary uploads are streamed by the authorized route into a bounded + # temporary file; do not copy a 10 MiB body into every API request. + if scope['method']=='POST' and scope['path'].endswith('/attachments'): + async with asyncio.timeout(self.cfg.request_timeout): + await self.app(scope,receive,tracked_send) + return + chunks, total = bytearray(), 0 + async with asyncio.timeout(self.cfg.request_timeout): + while True: + message = await receive() + if message['type'] == 'http.disconnect': + return + chunk = message.get('body', b'') + total += len(chunk) + if total > self.cfg.max_body_bytes: + raise ApiError(413, 'payload_too_large', 'Request body exceeds byte limit') + chunks.extend(chunk) + if not message.get('more_body', False): + break + body = bytes(chunks) + delivered = False + async def replay(): + nonlocal delivered + if delivered: + return await receive() + delivered = True + return {'type': 'http.request', 'body': body, 'more_body': False} + await self.app(scope, replay, tracked_send) + except ClientDisconnect: + status=499 + except TimeoutError: + if not started_response: + await error_response(ApiError(503, 'unavailable', 'Request timed out'), request_id)(scope, receive, tracked_send) + except ApiError as exc: + if not started_response: + await error_response(exc, request_id)(scope, receive, tracked_send) + except Exception as exc: + logger.error(json.dumps({'event': 'unexpected_error', 'request_id': request_id, 'type': type(exc).__name__})) + if not started_response: + await error_response(ApiError(500, 'internal_error', 'Internal error'), request_id)(scope, receive, tracked_send) + finally: + if admitted: + self.active -= 1 + # No query, body, headers, usernames, or message content in logs. + route = scope.get('route') + logger.info(json.dumps({'event': 'http', 'request_id': request_id, 'method': scope['method'], + 'route': getattr(route, 'path', 'unmatched'), 'status': status, + 'duration_ms': round((time.monotonic() - started) * 1000, 3)})) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..adbf929d --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,54 @@ +"""Job state, transactional outbox and explicit operator transitions.""" +from datetime import datetime, timezone +import os +from uuid import uuid4 + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +MAX_ATTEMPTS = 5 +LEASE_SECONDS = 60 + + +def database(): + return psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, + connect_timeout=3, autocommit=True, + options='-c statement_timeout=5000 -c lock_timeout=1000') + + +def audit(db, job_id, event, **details): + db.execute('INSERT INTO job_audit(job_id,event,details) VALUES (%s,%s,%s)', + (job_id, event, Jsonb(details))) + + +def payload(job): + result = {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', + 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], + 'attempt': job['attempts'] + 1, + 'created_at': job['created_at'].astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')} + if job.get('traceparent'): + result['traceparent'] = job['traceparent'] + return result + + +def recover_expired(db): + with db.transaction(): + rows = db.execute("SELECT * FROM jobs WHERE status='processing' AND lease_until= MAX_ATTEMPTS + status = 'failed' if exhausted else 'queued' + db.execute("UPDATE jobs SET status=%s,lease_token=NULL,lease_until=NULL,error_code=%s,next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=%s", + (status, 'retry_exhausted' if exhausted else 'lease_expired', job['id'])) + db.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s AND status='processing'", + (status, 'retry_exhausted' if exhausted else None, job['attachment_id'], job['pipeline_version'])) + db.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s', (job['id'],)) + audit(db, job['id'], 'lease_expired', attempt=job['attempts'], outcome=status) + + +def take_fault(db, job_id): + # Only the operator test harness can populate this table. No HTTP fault switches. + with db.transaction(): + row = db.execute('UPDATE job_faults SET remaining=remaining-1 WHERE job_id=%s AND remaining>0 RETURNING point', + (job_id,)).fetchone() + return row['point'] if row else None diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..a66ac916 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,28 @@ +"""Versioned, transactional migrations, serialized across deployers.""" +import hashlib +import os +from pathlib import Path +import psycopg + + +def main(): + with psycopg.connect(os.environ['DATABASE_URL'],autocommit=True,connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_lock(23012026)') + try: + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())') + for file in sorted((Path(__file__).resolve().parents[1]/'migrations').glob('*.sql')): + data=file.read_bytes(); digest=hashlib.sha256(data).hexdigest() + row=conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s',(file.name,)).fetchone() + if row: + if row[0]!=digest: raise RuntimeError('An applied migration was modified: '+file.name) + print('Already applied:',file.name,flush=True) + continue + with conn.transaction(): + conn.execute(data.decode('utf-8')) + conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)',(file.name,digest)) + print('Applied:',file.name,flush=True) + finally: + conn.execute('SELECT pg_advisory_unlock(23012026)') + + +if __name__=='__main__': main() diff --git a/src/offload.py b/src/offload.py new file mode 100644 index 00000000..24842d33 --- /dev/null +++ b/src/offload.py @@ -0,0 +1,16 @@ +"""Keep resource leases until a blocking operation actually finishes on cancellation.""" +import asyncio + + +async def complete_thread(function, *args): + task = asyncio.create_task(asyncio.to_thread(function, *args)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + # asyncio cancellation cannot stop a Python thread. Do not close its file + # or release its decoder/upload permit while it is still using them. + try: + await asyncio.shield(task) + except Exception: + pass + raise diff --git a/src/operator.py b/src/operator.py new file mode 100644 index 00000000..ce381bf3 --- /dev/null +++ b/src/operator.py @@ -0,0 +1,71 @@ +"""Local operator CLI: failed registry, audited replay and versioned reprocessing.""" +import argparse +import json +import tempfile +from uuid import UUID, uuid4 + +from .jobs import audit, database +from .storage import MAX_BYTES, Storage + + +def replay(job_id): + storage=Storage() + try: + with database() as db, db.transaction(): + job=db.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone() + if not job or job['status']!='failed': raise ValueError('Only failed jobs can be replayed') + att=db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone() + if att['pipeline_version']!=job['pipeline_version'] or not att['staging_key']: + raise ValueError('Stale version or expired staging; replay refused') + storage.client.head_object(Bucket=storage.bucket,Key=att['staging_key']) + db.execute("UPDATE jobs SET status='queued',attempts=0,error_code=NULL,lease_token=NULL,lease_until=NULL,next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=%s",(job_id,)) + db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(att['id'],)) + db.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job_id,)) + audit(db,job_id,'operator_replay',previous_attempts=job['attempts']) + return {'job_id':str(job_id),'status':'queued'} + finally: storage.client.close() + + +def upgrade(attachment_id,version): + if version not in (1,2): raise ValueError('Supported pipeline versions are 1 and 2') + storage=Storage() + try: + with database() as db, db.transaction(): + att=db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(attachment_id,)).fetchone() + if not att or version<=att['pipeline_version']: raise ValueError('Version must strictly increase') + source=att['object_key'] if att['status']=='ready' else att['staging_key'] + if not source: raise ValueError('Original is no longer available') + key=f'staging/{attachment_id}/v{version}' + db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,)) + obj=storage.get(source) + try: + with tempfile.TemporaryFile() as file: + size=0 + while block:=obj['Body'].read(65536): + size+=len(block) + if size>MAX_BYTES: raise ValueError('Original exceeds byte limit') + file.write(block) + storage.put(key,file,size,att['content_type'],att['sha256']) + finally: obj['Body'].close() + job_id=uuid4() + db.execute("UPDATE attachments SET pipeline_version=%s,status='queued',staging_key=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(version,key,attachment_id)) + db.execute('INSERT INTO jobs(id,attachment_id,pipeline_version) VALUES (%s,%s,%s)',(job_id,attachment_id,version)) + db.execute('INSERT INTO outbox(job_id) VALUES (%s)',(job_id,)) + audit(db,job_id,'operator_upgrade',previous_version=att['pipeline_version'],version=version) + return {'job_id':str(job_id),'pipeline_version':version,'status':'queued'} + finally: storage.client.close() + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + commands=parser.add_subparsers(dest='command',required=True) + commands.add_parser('failed') + commands.add_parser('replay').add_argument('job_id',type=UUID) + p=commands.add_parser('upgrade'); p.add_argument('attachment_id',type=UUID); p.add_argument('version',type=int) + args=parser.parse_args() + if args.command=='failed': + with database() as db: + result=db.execute("SELECT id,attachment_id,pipeline_version,attempts,error_code,updated_at FROM jobs WHERE status='failed' ORDER BY updated_at DESC LIMIT 100").fetchall() + elif args.command=='replay': result=replay(args.job_id) + else: result=upgrade(args.attachment_id,args.version) + print(json.dumps(result,default=str,indent=2)) diff --git a/src/render.py b/src/render.py new file mode 100644 index 00000000..08b841c6 --- /dev/null +++ b/src/render.py @@ -0,0 +1,69 @@ +"""Isolated image CPU/memory workload. Input/output paths are operator-owned temporary files.""" +import hashlib +import json +from pathlib import Path +import resource +import sys + +from PIL import Image, ImageOps + + +def fit_size(width, height, bound): + ratio = min(1, bound / width, bound / height) + return max(1, int(width * ratio)), max(1, int(height * ratio)) + + +def render(source, destination, version): + # Both versions are explicit, reproducible pipelines. Version 2 changes quality; + # storage effect keys include version, so old jobs cannot overwrite new bytes. + quality = {1: (85, 80), 2: (90, 85)}[version] + Image.MAX_IMAGE_PIXELS = 40_000_000 + with Image.open(source, formats=['PNG', 'JPEG']) as opened: + if opened.width * opened.height > 40_000_000: + raise ValueError('pixel_limit') + opened.load() + image = ImageOps.exif_transpose(opened) + try: + if image.mode in ('RGBA', 'LA') or 'transparency' in image.info: + rgba = image.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + rgba.close() + else: + rgb = image.convert('RGB') + finally: + image.close() + rgb.info.clear() # Do not propagate EXIF, ICC or arbitrary image metadata. + result = [] + try: + for name, mime, fmt in [('thumbnail', 'image/jpeg', 'JPEG'), + ('crop', 'image/jpeg', 'JPEG'), + ('optimized', 'image/webp', 'WEBP')]: + if name == 'crop': + variant = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, + centering=(.5, .5)) + else: + variant = rgb.resize(fit_size(*rgb.size, 256 if name == 'thumbnail' else 1280), + Image.Resampling.LANCZOS) + try: + variant.info.clear() + target = destination / name + options = {'quality': quality[1], 'method': 4} if fmt == 'WEBP' else { + 'quality': quality[0], 'optimize': True, 'subsampling': 0} + variant.save(target, format=fmt, **options) + result.append({'name': name, 'content_type': mime, 'width': variant.width, + 'height': variant.height, 'size_bytes': target.stat().st_size, + 'sha256': hashlib.sha256(target.read_bytes()).hexdigest()}) + finally: + variant.close() + finally: + rgb.close() + return result + + +if __name__ == '__main__': + # Container is the outer memory boundary; subprocess also has finite virtual + # memory and CPU budgets. Parent enforces a wall-clock timeout and kills it. + resource.setrlimit(resource.RLIMIT_AS, (640 * 1024 * 1024, 640 * 1024 * 1024)) + resource.setrlimit(resource.RLIMIT_CPU, (20, 20)) + print(json.dumps(render(Path(sys.argv[1]), Path(sys.argv[2]), int(sys.argv[3])))) diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..71d90730 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,74 @@ +"""Bounded S3 calls and strict PNG/JPEG validation; never trust a client filename.""" +import hashlib +import os +import warnings + +import boto3 +from botocore.config import Config +from PIL import Image, UnidentifiedImageError + +from .common import ApiError + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +HEALTH_KEY = 'originals/.storage-health' +HEALTH_BYTES = b'messenger-storage-ready-v1' +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +warnings.simplefilter('error', Image.DecompressionBombWarning) + + +def validate_image(file, content_type): + try: + file.seek(0) + with Image.open(file, formats=['PNG', 'JPEG']) as image: + actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}[image.format] + if actual != content_type: + raise ApiError(415, 'unsupported_media_type', 'Image content does not match MIME') + if image.width * image.height > MAX_PIXELS: + raise ApiError(413, 'payload_too_large', 'Too many decoded pixels') + image.verify() + file.seek(0) + with Image.open(file, formats=['PNG', 'JPEG']) as image: + image.load() # JPEG.verify alone does not detect a truncated entropy stream. + file.seek(0) + digest = hashlib.file_digest(file, 'sha256').hexdigest() + file.seek(0) + return digest + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise ApiError(413, 'payload_too_large', 'Too many decoded pixels') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise ApiError(415, 'unsupported_media_type', 'Invalid PNG/JPEG') from None + + +class Storage: + def __init__(self): + self.bucket = os.environ['S3_BUCKET'] + self.client = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'], + aws_access_key_id=os.environ['S3_ACCESS_KEY'], + aws_secret_access_key=os.environ['S3_SECRET_KEY'], + region_name='us-east-1', + config=Config(connect_timeout=2, read_timeout=4, + retries={'total_max_attempts': 1}, + max_pool_connections=8, + s3={'addressing_style': 'path'})) + + def ready(self): + # Listing proves only filer metadata readiness. After a SeaweedFS restart, + # volume registration can lag; require a real read from a persisted volume. + obj = self.get(HEALTH_KEY) + try: + if obj['Body'].read(len(HEALTH_BYTES)+1) != HEALTH_BYTES: + raise ApiError(503, 'unavailable', 'Storage probe unavailable') + finally: + obj['Body'].close() + + def put(self, key, file, size, content_type, digest): + file.seek(0) + self.client.put_object(Bucket=self.bucket, Key=key, Body=file, ContentLength=size, + ContentType=content_type, Metadata={'sha256': digest}) + + def get(self, key): + return self.client.get_object(Bucket=self.bucket, Key=key) + + def delete(self, key): + self.client.delete_object(Bucket=self.bucket, Key=key) diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..f67df731 --- /dev/null +++ b/src/store.py @@ -0,0 +1,174 @@ +import hashlib +import asyncio +import json +import logging +from uuid import uuid4 + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from .common import ApiError, encode_cursor, invalid + + +def clean(row): + if isinstance(row,dict): return {k:clean(v) for k,v in row.items()} + if isinstance(row,list): return [clean(v) for v in row] + if hasattr(row,'isoformat'): return row.isoformat(timespec='microseconds').replace('+00:00','Z') + if type(row).__name__=='UUID': return str(row) + return row + + +class Store: + def __init__(self,cfg): + self.cfg=cfg + self.secret=cfg.cursor_secret.encode() + self.pool=AsyncConnectionPool(cfg.database_url,min_size=1,max_size=cfg.db_pool_size, + timeout=2,max_waiting=32,open=False, + kwargs={'row_factory':dict_row,'connect_timeout':3, + 'options':'-c statement_timeout=2000 -c lock_timeout=1000 -c timezone=UTC'}) + self.cache=Redis.from_url(cfg.redis_url,socket_timeout=.2,socket_connect_timeout=.2, + max_connections=8,decode_responses=True) + + async def open(self): await self.pool.open(wait=True,timeout=20) + async def close(self): + await self.cache.aclose() + await self.pool.close() + + async def ready(self): + async with self.pool.connection() as c: await c.execute('SELECT 1') + + async def register(self,username,display_name,password_hash): + try: + async with self.pool.connection() as c: + row=await (await c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at', + (uuid4(),username,display_name,password_hash))).fetchone() + return clean(row) + except psycopg.errors.UniqueViolation: + raise ApiError(409,'conflict','Username already exists') from None + + async def credentials(self,username): + async with self.pool.connection() as c: + return await (await c.execute('SELECT * FROM users WHERE username=%s',(username,))).fetchone() + + async def create_session(self,user_id,token): + async with self.pool.connection() as c: + row=await (await c.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp()+%s*interval '1 second') RETURNING expires_at", + (hashlib.sha256(token.encode()).hexdigest(),user_id,self.cfg.session_ttl_seconds))).fetchone() + return clean(row)['expires_at'] + + async def authenticate(self,token): + async with self.pool.connection() as c: + session=await (await c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()', + (hashlib.sha256(token.encode()).hexdigest(),))).fetchone() + if session is None: raise ApiError(401,'unauthorized','Invalid session',{'WWW-Authenticate':'Bearer realm="messenger"'}) + key='user:'+str(session['user_id']) + try: + cached=await self.cache.get(key) + if cached: return json.loads(cached) + except (RedisError,ValueError): + logging.getLogger('messenger').warning('{"event":"cache_bypass"}') + row=await (await c.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s',(session['user_id'],))).fetchone() + user=clean(row) + try: await self.cache.set(key,json.dumps(user),ex=60) + except RedisError: pass + return user + + async def logout(self,token): + async with self.pool.connection() as c: + await c.execute('DELETE FROM sessions WHERE token_hash=%s',(hashlib.sha256(token.encode()).hexdigest(),)) + + async def _chat(self,c,actor,chat_id): + row=await (await c.execute('SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY user_id) AS member_ids FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s',(chat_id,actor))).fetchone() + if row is None: raise ApiError(404,'not_found','Object not found') + return clean(row) + + async def chat(self,actor,chat_id): + async with self.pool.connection() as c: return await self._chat(c,actor,chat_id) + + async def create_chat(self,actor,title,members): + if actor not in members: invalid('Members must include the current user') + try: + async with self.pool.connection() as c: + cid=uuid4() + await c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(cid,title)) + async with c.cursor() as cur: + await cur.executemany('INSERT INTO members(chat_id,user_id) VALUES (%s,%s)',[(cid,m) for m in members]) + return await self._chat(c,actor,cid) + except psycopg.errors.ForeignKeyViolation: + invalid('All members must exist') + + async def create_message(self,actor,chat_id,text,key,attachment_ids): + digest=hashlib.sha256(json.dumps({'text':text,'attachment_ids':attachment_ids},sort_keys=True,ensure_ascii=False,separators=(',',':')).encode()).hexdigest() + async with self.pool.connection() as c: + await self._chat(c,actor,chat_id) + # Fixed operation POST /chats/{id}/messages; lock and both rows share one transaction. + scope='POST:'+chat_id+':'+actor+':'+key + await c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,)) + previous=await (await c.execute("SELECT request_hash,message_id FROM idempotency WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()",(actor,chat_id,key))).fetchone() + if previous: + if previous['request_hash']!=digest: raise ApiError(409,'conflict','Key was used for another message') + return clean(await (await c.execute('SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE id=%s',(previous['message_id'],))).fetchone()) + for aid in attachment_ids: + attachment=await (await c.execute('SELECT status,storage_state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE',(aid,chat_id))).fetchone() + if attachment is None: raise ApiError(404,'not_found','Object not found') + if attachment['status']=='failed': raise ApiError(409,'conflict','Attachment processing failed') + if attachment['status']=='ready' and attachment['storage_state']!='ready': raise ApiError(503,'unavailable','Object temporarily unavailable') + await c.execute('DELETE FROM idempotency WHERE actor_id=%s AND chat_id=%s AND key=%s',(actor,chat_id,key)) + row=await (await c.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING id,chat_id,sender_id,text,created_at,attachment_ids', + (uuid4(),chat_id,actor,text,attachment_ids))).fetchone() + for aid in attachment_ids: + await c.execute('INSERT INTO message_attachments(message_id,attachment_id) VALUES (%s,%s)',(row['id'],aid)) + await c.execute("INSERT INTO idempotency(actor_id,chat_id,key,request_hash,message_id,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",(actor,chat_id,key,digest,row['id'])) + return clean(row) + + async def page(self,actor,chat_id,limit,cursor): + async with self.pool.connection() as c: + if chat_id: + await self._chat(c,actor,chat_id) + query='SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE chat_id=%s' + args=[chat_id]; resource='messages:'+chat_id + if cursor: + query+=' AND (created_at,id)>(%s::timestamptz,%s::uuid)'; args.extend(cursor) + query+=' ORDER BY created_at,id LIMIT %s'; args.append(limit+1) + else: + query='SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY user_id) AS member_ids FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s' + args=[actor]; resource='chats:'+actor + if cursor: + query+=' AND (c.created_at,c.id)>(%s::timestamptz,%s::uuid)'; args.extend(cursor) + query+=' ORDER BY c.created_at,c.id LIMIT %s'; args.append(limit+1) + rows=[clean(row) for row in await (await c.execute(query,args)).fetchall()] + selected=rows[:limit] + next_cursor=encode_cursor(resource,(selected[-1]['created_at'],selected[-1]['id']),self.secret) if len(rows)>limit else None + return {'items':selected,'next_cursor':next_cursor} + + async def attachment(self,actor,identifier): + async with self.pool.connection() as c: + row=await (await c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(identifier,actor))).fetchone() + if row is None: raise ApiError(404,'not_found','Object not found') + if row['status']=='ready' and row['storage_state']!='ready': raise ApiError(503,'unavailable','Object temporarily unavailable') + return clean(row) + + async def save_attachment(self,actor,chat_id,filename,mime,size,digest,file,storage): + identifier=uuid4() + key='staging/'+str(identifier)+'/v1' + # The same lock is taken by reconciliation before it can delete an orphan. + # DB COMMIT uncertainty must NEVER trigger an unconditional S3 delete: the row + # may have committed. Leave a possible orphan for the grace-period reconciler. + async with self.pool.connection() as c: + await self._chat(c,actor,chat_id) + # Admission is global and serialized: no unbounded durable backlog. + await c.execute("SELECT pg_advisory_xact_lock(hashtextextended('job-admission',0))") + pending=await (await c.execute("SELECT count(*) AS n FROM jobs WHERE status IN ('queued','processing')")).fetchone() + if pending['n']>=1000: raise ApiError(503,'unavailable','Processing backlog is full') + await c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,)) + from .offload import complete_thread + await complete_thread(storage.put,key,file,size,mime,digest) + row=await (await c.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,status,staging_key) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) RETURNING *", + (identifier,chat_id,actor,filename,key,mime,size,digest,key))).fetchone() + job_id=uuid4() + await c.execute('INSERT INTO jobs(id,attachment_id,pipeline_version) VALUES (%s,%s,1)',(job_id,identifier)) + await c.execute('INSERT INTO outbox(job_id) VALUES (%s)',(job_id,)) + return clean(row) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..df44bd84 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,210 @@ +"""At-least-once processing with versioned effects, fenced commits and bounded retries.""" +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import random +import signal +import subprocess +import sys +import tempfile +import threading +import time +from uuid import UUID, uuid4 + +from botocore.exceptions import ClientError +from psycopg.types.json import Jsonb + +from .broker import acknowledge, connect, receive +from .jobs import MAX_ATTEMPTS, audit, database, take_fault +from .storage import MAX_BYTES, Storage + +stop = threading.Event() + + +class Permanent(Exception): + pass + + +def crash(point, job_id, code): + print(json.dumps({'event':'injected_crash','point':point,'job_id':str(job_id),'exit_code':code}),flush=True) + os._exit(code) + + +def parse(raw): + item = json.loads(raw) + required = {'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'} + if set(item)-required-{'traceparent','tracestate'} or required-set(item): + raise ValueError('invalid_job_fields') + if item['schema_version'] != 1 or item['type'] != 'attachment.process.v1': + raise ValueError('unsupported_job') + UUID(item['job_id']); UUID(item['attachment_id']) + for field in ('attempt','pipeline_version'): + if type(item[field]) is not int or item[field]<1: + raise ValueError('invalid_job_number') + datetime.fromisoformat(item['created_at'].replace('Z','+00:00')) + return item + + +def claim(db, item): + with db.transaction(): + job = db.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE', (item['job_id'],)).fetchone() + if not job or job['status'] in ('done','failed','superseded'): + return None + if str(job['attachment_id'])!=item['attachment_id'] or job['pipeline_version']!=item['pipeline_version']: + return None + attachment = db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (job['attachment_id'],)).fetchone() + if attachment['pipeline_version']!=job['pipeline_version']: + db.execute("UPDATE jobs SET status='superseded',updated_at=clock_timestamp() WHERE id=%s", (job['id'],)) + audit(db, job['id'], 'stale_version_ignored') + return None + if job['status']!='queued' or job['next_attempt_at']>datetime.now(timezone.utc): + return None + if item['attempt']!=job['attempts']+1: + return None + token=uuid4() + job=db.execute("""UPDATE jobs SET status='processing',attempts=attempts+1,lease_token=%s, + lease_until=clock_timestamp()+interval '60 seconds',updated_at=clock_timestamp() + WHERE id=%s RETURNING *""", (token,job['id'])).fetchone() + db.execute("UPDATE attachments SET status='processing',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (attachment['id'],)) + audit(db,job['id'],'claimed',attempt=job['attempts'],version=job['pipeline_version']) + return job,attachment + + +def process_bytes(storage, attachment, version, folder): + source=folder/'source' + if not attachment['staging_key']: + raise Permanent('staging_missing') + try: + obj=storage.get(attachment['staging_key']) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404','NoSuchKey'): + raise Permanent('staging_missing') from None + raise + size=0; digest=hashlib.sha256() + try: + with source.open('wb') as file: + while block:=obj['Body'].read(65536): + size+=len(block) + if size>MAX_BYTES: + raise Permanent('size_limit') + digest.update(block); file.write(block) + finally: + obj['Body'].close() + if size!=attachment['size_bytes'] or digest.hexdigest()!=attachment['sha256']: + raise Permanent('checksum_mismatch') + try: + rendering=subprocess.run([sys.executable,'-m','src.render',str(source),str(folder),str(version)], + capture_output=True,text=True,timeout=25) + except subprocess.TimeoutExpired: + raise Permanent('processing_timeout') from None + if rendering.returncode: + raise Permanent('invalid_image_or_resource_limit') + variants=json.loads(rendering.stdout) + # Version is part of every final key. Never use an unversioned mutable alias. + key=f'originals/{attachment["id"]}/v{version}' + with source.open('rb') as file: + storage.put(key,file,size,attachment['content_type'],attachment['sha256']) + for variant in variants: + variant['object_key']=f'variants/{attachment["id"]}/v{version}/{variant["name"]}' + with (folder/variant['name']).open('rb') as file: + storage.put(variant['object_key'],file,variant['size_bytes'],variant['content_type'],variant['sha256']) + return key,variants + + +def publish_result(db, job, key, variants): + with db.transaction(): + current=db.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE', (job['id'],)).fetchone() + att=db.execute('SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE', (job['attachment_id'],)).fetchone() + if current['status']!='processing' or current['lease_token']!=job['lease_token'] or current['lease_until']<=datetime.now(timezone.utc): + return False + if att['pipeline_version']!=job['pipeline_version']: + db.execute("UPDATE jobs SET status='superseded',lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",(job['id'],)) + audit(db,job['id'],'stale_commit_ignored') + return False + db.execute("""UPDATE attachments SET status='ready',storage_state='ready',object_key=%s, + variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s""", + (key,Jsonb(variants),job['attachment_id'])) + db.execute("UPDATE jobs SET status='done',error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",(job['id'],)) + audit(db,job['id'],'ready',attempt=job['attempts'],version=job['pipeline_version']) + return True + + +def failed_attempt(db, job, exc): + permanent=isinstance(exc,Permanent) + final=permanent or job['attempts']>=MAX_ATTEMPTS + code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_or_processing_unavailable') + delay=min(30,2**job['attempts'])+random.uniform(0,.5) + state='failed' if final else 'queued' + with db.transaction(): + row=db.execute("""UPDATE jobs SET status=%s,error_code=%s,lease_token=NULL,lease_until=NULL, + next_attempt_at=clock_timestamp()+(%s*interval '1 second'),updated_at=clock_timestamp() + WHERE id=%s AND status='processing' AND lease_token=%s RETURNING id""", + (state,code,delay,job['id'],job['lease_token'])).fetchone() + if not row: return + db.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s", + (state,code if final else None,job['attachment_id'],job['pipeline_version'])) + db.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],)) + audit(db,job['id'],state,error_code=code,attempt=job['attempts'],retry_delay_seconds=round(delay,3)) + + +def handle(item): + with database() as db: + claimed=claim(db,item) + if claimed is None: return 'noop' + job,attachment=claimed + fault=take_fault(db,job['id']) + if fault=='before_effect': crash(fault,job['id'],71) + storage=Storage() + try: + if fault=='transient': raise RuntimeError('synthetic_transient') + if fault=='permanent': raise Permanent('synthetic_permanent') + with tempfile.TemporaryDirectory(prefix='job-') as directory: + key,variants=process_bytes(storage,attachment,job['pipeline_version'],Path(directory)) + if fault=='after_effect': crash(fault,job['id'],72) + done=publish_result(db,job,key,variants) + if fault=='after_commit': crash(fault,job['id'],73) + # Cleanup is independently repeatable. Failure cannot roll back ready. + if done and attachment['staging_key']: + try: + storage.delete(attachment['staging_key']) + db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND pipeline_version=%s AND status='ready' AND staging_key=%s", + (attachment['id'],job['pipeline_version'],attachment['staging_key'])) + except Exception: + print(json.dumps({'event':'staging_cleanup_deferred'}),flush=True) + return 'ready' if done else 'superseded' + except Exception as exc: + failed_attempt(db,job,exc) + return 'failed' if isinstance(exc,Permanent) or job['attempts']>=MAX_ATTEMPTS else 'retry' + finally: + storage.client.close() + + +def main(): + signal.signal(signal.SIGTERM,lambda *_:stop.set()) + signal.signal(signal.SIGINT,lambda *_:stop.set()) + consumer='worker-'+uuid4().hex + while not stop.is_set(): + try: + with connect() as client: + entry=receive(client,consumer) + if not entry: continue + message_id,fields=entry + try: item=parse(fields.get('job','')) + except (ValueError,KeyError,TypeError): + print(json.dumps({'event':'invalid_broker_payload'}),flush=True) + acknowledge(client,message_id) + continue + started=time.monotonic() + outcome=handle(item) + acknowledge(client,message_id) + print(json.dumps({'event':'job_finished','job_id':item['job_id'],'outcome':outcome, + 'seconds':round(time.monotonic()-started,6)}),flush=True) + except Exception as exc: + # No ack on uncertain DB/transport result; lease/outbox re-drive intent. + print(json.dumps({'event':'worker_retry','type':type(exc).__name__}),flush=True) + stop.wait(1) + + +if __name__=='__main__': main() -- 2.54.0 From 06c541692c12d69a40ad42194a48c5d21df1516d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BB=D0=B5=D0=B1=20=D0=9F=D0=B0=D1=80=D0=B0=D0=BC?= =?UTF-8?q?=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Sat, 17 Oct 2026 03:07:53 +0300 Subject: [PATCH 2/3] Record successful clean clone recovery experiments --- REPORT.md | 2 + evidence/clean-clone.json | 177 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 evidence/clean-clone.json diff --git a/REPORT.md b/REPORT.md index 08baf7cf..4a34727e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -77,6 +77,8 @@ Replay только failed текущей версии при наличии sta ## Ограничения и источники +Повтор из чистого Git clone с новыми secrets и volumes также PASS: `evidence/clean-clone.json`. Повторены весь smoke/persistence/failover и все fault/retry/version cases. Crash recovery70.296/71.031/14.000s; failover108 успешных записей,2 failed attempts, recovery5.000s. Это независимый повтор bootstrap, не запуск поверх прежних данных. + Один хост и одиночные PG/S3/broker; потерю их volumes решение не маскирует. Изоляция памяти ограничивает допустимые изображения сверх простого byte/pixel лимита. Lease не продлевается; слишком медленный эффект отклоняется fencing и повторяется ограниченно. Старые versioned результаты сохранены для известных attachments; автоматическая политика удаления истории версий не реализована. Научный вывод ограничен перечисленными точками отказа, не формальным доказательством всех interleavings. Источники: [Valkey persistence](https://valkey.io/topics/persistence/), [XREADGROUP](https://valkey.io/commands/xreadgroup/), [XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [Pillow](https://pillow.readthedocs.io/en/stable/), контракты этого репозитория. Реализация, сценарии, анализ и оформление подготовлены с помощью OpenAI Codex; фактические результаты получены локальными прогонами, помощь ИИ не скрывается. diff --git a/evidence/clean-clone.json b/evidence/clean-clone.json new file mode 100644 index 00000000..6ef78e18 --- /dev/null +++ b/evidence/clean-clone.json @@ -0,0 +1,177 @@ +{ + "public_smoke": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.", + "commit_before_publish": { + "attachment_id": "d1a1fa85-161a-4423-b69e-554823a3a967", + "job_id": "8bac3ff1-93f4-4436-8d9a-90ca64aa92f6", + "killed_api": "33dafd306436075b5b8d927a8d376aae1b84ccd2a26f4b66efa6fa2d0107b360", + "broker_restarted": true, + "queued_message_accepted": true, + "original_and_variants_verified": true + }, + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 108, \"failed_attempts\": 2, \"first_recovery_seconds\": 5.0}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "terminal_duplicates": { + "deliveries": 3, + "effect_objects": 4, + "metadata_unchanged": true + }, + "crashes": [ + { + "point": "before_effect", + "seconds_to_ready": 70.296, + "attempts": 2, + "effect_objects": 4, + "audit": [ + { + "event": "claimed", + "details": { + "attempt": 1, + "version": 1 + } + }, + { + "event": "lease_expired", + "details": { + "attempt": 1, + "outcome": "queued" + } + }, + { + "event": "claimed", + "details": { + "attempt": 2, + "version": 1 + } + }, + { + "event": "ready", + "details": { + "attempt": 2, + "version": 1 + } + } + ], + "crash_event": { + "event": "injected_crash", + "point": "before_effect", + "job_id": "269f3fa1-3430-4e36-b030-91fdb4e60c5c", + "exit_code": 71 + }, + "restart_count_before": 0, + "restart_count_after": 1 + }, + { + "point": "after_effect", + "seconds_to_ready": 71.031, + "attempts": 2, + "effect_objects": 4, + "audit": [ + { + "event": "claimed", + "details": { + "attempt": 1, + "version": 1 + } + }, + { + "event": "lease_expired", + "details": { + "attempt": 1, + "outcome": "queued" + } + }, + { + "event": "claimed", + "details": { + "attempt": 2, + "version": 1 + } + }, + { + "event": "ready", + "details": { + "attempt": 2, + "version": 1 + } + } + ], + "crash_event": { + "event": "injected_crash", + "point": "after_effect", + "job_id": "7475f576-a43d-49bd-95cc-f30fe5037844", + "exit_code": 72 + }, + "restart_count_before": 1, + "restart_count_after": 1 + }, + { + "point": "after_commit", + "seconds_to_ready": 14.0, + "attempts": 1, + "effect_objects": 4, + "audit": [ + { + "event": "claimed", + "details": { + "attempt": 1, + "version": 1 + } + }, + { + "event": "ready", + "details": { + "attempt": 1, + "version": 1 + } + } + ], + "crash_event": { + "event": "injected_crash", + "point": "after_commit", + "job_id": "17161f10-5c93-411b-9347-11e4e7d9fab4", + "exit_code": 73 + }, + "restart_count_before": 1, + "restart_count_after": 1 + } + ], + "retries": [ + { + "fault": "transient", + "injected": 2, + "terminal_status": "ready", + "error_code": null, + "attempts": 3, + "replay_ready": false + }, + { + "fault": "permanent", + "injected": 1, + "terminal_status": "failed", + "error_code": "synthetic_permanent", + "attempts": 1, + "replay_ready": true + }, + { + "fault": "transient", + "injected": 5, + "terminal_status": "failed", + "error_code": "retry_exhausted", + "attempts": 5, + "replay_ready": true + } + ], + "version_reorder": { + "old_job_id": "9bf005bb-4112-4ba9-afac-5b0808ad4fbd", + "new_job_id": "d43c9e91-1af8-47bf-ae32-74f94f425e9b", + "final_pipeline_version": 2, + "keys": [ + "originals/5bc7e699-5e91-47eb-a99c-16928c21a790/v2", + "variants/5bc7e699-5e91-47eb-a99c-16928c21a790/v2/crop", + "variants/5bc7e699-5e91-47eb-a99c-16928c21a790/v2/optimized", + "variants/5bc7e699-5e91-47eb-a99c-16928c21a790/v2/thumbnail" + ], + "stale_delivery_did_not_change_result": true + }, + "passed": true +} -- 2.54.0 From 0c022e633c9b30f5aa73def8dd57f16bb54a9382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BB=D0=B5=D0=B1=20=D0=9F=D0=B0=D1=80=D0=B0=D0=BC?= =?UTF-8?q?=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Fri, 23 Oct 2026 13:23:06 +0300 Subject: [PATCH 3/3] Add isolated application build and smoke CI job --- .github/workflows/application.yml | 25 +++++++++++++++++++++++++ REPORT.md | 4 ++++ 2 files changed, 29 insertions(+) create mode 100644 .github/workflows/application.yml diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..bdc72e2c --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,25 @@ +name: Application smoke +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + # Run on an isolated disposable runner with Docker Compose v2, never a shared production host. + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: mtusi-ci + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-dev.txt + - run: make check validate PYTHON=python + - run: python scripts/bootstrap.py + - run: docker compose config --quiet + - run: docker compose up -d --build --wait --wait-timeout 180 + - run: make test PYTHON=python + - name: Stop this disposable test stack + if: always() + run: docker compose down --volumes --remove-orphans diff --git a/REPORT.md b/REPORT.md index 4a34727e..3aab7b78 100644 --- a/REPORT.md +++ b/REPORT.md @@ -82,3 +82,7 @@ Replay только failed текущей версии при наличии sta Один хост и одиночные PG/S3/broker; потерю их volumes решение не маскирует. Изоляция памяти ограничивает допустимые изображения сверх простого byte/pixel лимита. Lease не продлевается; слишком медленный эффект отклоняется fencing и повторяется ограниченно. Старые versioned результаты сохранены для известных attachments; автоматическая политика удаления истории версий не реализована. Научный вывод ограничен перечисленными точками отказа, не формальным доказательством всех interleavings. Источники: [Valkey persistence](https://valkey.io/topics/persistence/), [XREADGROUP](https://valkey.io/commands/xreadgroup/), [XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [Pillow](https://pillow.readthedocs.io/en/stable/), контракты этого репозитория. Реализация, сценарии, анализ и оформление подготовлены с помощью OpenAI Codex; фактические результаты получены локальными прогонами, помощь ИИ не скрывается. + +## Автоматическая проверка приложения + +`.github/workflows/application.yml` добавляет отдельный job: проверка контрактов, bootstrap (когда нужен), сборка, ожидание readiness и public smoke. Он предназначен для изолированного одноразового Actions-compatible Linux runner с Docker Compose v2; secrets создаются внутри job, после проверки удаляются только его Compose volumes. На GitFlic/Gitea запуск зависит от настроек платформы и runner преподавателя. Успешный удалённый CI этим отчётом не заявляется: приведённые доказательства получены локально, включая чистый clone. Полные fault/load эксперименты воспроизводятся отдельными командами отчёта. -- 2.54.0