diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..46526f6a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.png binary +*.jpg binary diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..a1e9ca91 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,11 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Парамонов Глеб, БВТ2301. Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Собственная ЛР3 дополнена приватным SeaweedFS S3, потоковыми PNG/JPEG uploads, +ACL вложений, авторизованным download, отдельными S3 credentials/bootstrap, +лимитами памяти и периодическим reconciler с dry-run и grace period. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `python scripts/bootstrap.py`, `docker compose up -d --build --wait`. +Команды проверок, наблюдения, исследование, источники и ограничения — в REPORT.md. +Контракт и public tests не изменены. Секреты вне Git, использование OpenAI Codex раскрыто. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..bdc72e2c --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,25 @@ +name: Application smoke +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + # Run on an isolated disposable runner with Docker Compose v2, never a shared production host. + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: mtusi-ci + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-dev.txt + - run: make check validate PYTHON=python + - run: python scripts/bootstrap.py + - run: docker compose config --quiet + - run: docker compose up -d --build --wait --wait-timeout 180 + - run: make test PYTHON=python + - name: Stop this disposable test stack + if: always() + run: docker compose down --volumes --remove-orphans diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..98c8e7d7 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,75 @@ +# План хранения и транзакций + +PostgreSQL — источник истины для пользователей, сессий, чатов, членства и сообщений. +Данные пользователей неизменяемы в обязательном API. Valkey кэширует именно их +публичное представление на 60 секунд; действительность сессии проверяется в PostgreSQL +при каждом запросе. Поэтому отказ кэша не отменяет logout/expiry и не даёт доступ +анонимному клиенту. При отказе PostgreSQL защищённые запросы получают 503. + +users: UUID PK, UNIQUE username, display_name, password_hash (Argon2id), created_at. +sessions: SHA-256 случайного 256-битного токена PK, user FK, expires_at, created_at. +chats: UUID PK, title, created_at. +members: (chat_id, user_id) PK и внешние ключи на chats/users. +messages: UUID PK, chat FK, sender FK, text, created_at; составной FK на members +не позволяет записать отправителя вне чата даже при ошибке в приложении. + +Создание чата и всех его участников — одна транзакция. Создание сообщения — одна +транзакция после проверки доступа. Регистрация защищена UNIQUE, гонка завершается +одним 201 и остальными 409. Logout удаляет сессию в транзакции; Valkey не хранит +разрешение на вход. Ранее начавшийся запрос может завершиться одновременно с logout. + +История: индекс (chat_id, created_at, id), keyset по (created_at,id), LIMIT n+1. +Отдельный индекс members(user_id,chat_id) ускоряет список доступных чатов. Часы БД +единые, но страницы между запросами не являются snapshot: конкурентная незавершённая +транзакция может стать видимой позднее курсора. Для неизменной истории пропусков нет. + +Пул ограничен 5 соединениями, ожидание 2 секунды; statement_timeout=2s, +lock_timeout=1s. KDF ограничен четырьмя параллельными задачами. Cursor подписан +общим локально генерируемым секретом, который сохраняется при рестарте. + +Миграции нумеруются, учитываются в schema_migrations, защищены advisory lock. +Отдельный одноразовый сервис migrate запускается до API. Повторный запуск проверяет +хеши применённых миграций, не пересоздаёт таблицы и не seed-ит данные. + +## Вложения и согласованность двух хранилищ + +SeaweedFS хранит байты в приватном bucket messenger. API имеет только +Read/Write для messenger/originals/* и List для messenger; отдельные административные +credentials доступны одноразовому storage-init. В S3-конфигурации нет anonymous. +API и S3 не публикуют host ports; filer/master/volume слушают loopback внутри S3-контейнера. + +Проверка Bearer/ACL и filename выполняется до чтения файла. Два upload slots на API +ограничивают одновременные временные файлы, один decoder slot — распаковку PNG/JPEG. +Оригинал пишется порциями в tmpfs /tmp (32 MiB); каждый файл ≤10 MiB. Проверка размеров +до декодирования ограничивает изображение 40 миллионами пикселей. API имеет 512 MiB RAM. +Временный файл — буфер текущего запроса, а не хранилище подтверждённого вложения. + +Ключ originals/UUID генерирует сервер. После валидации API берёт advisory xact lock +от ключа, пишет S3, затем INSERT метаданных в той же PostgreSQL-транзакции. 201 +возвращается после COMMIT. DB/S3 не имеют общей атомарной транзакции: падение после PUT +может оставить orphan. При неопределённом результате COMMIT запрещено безусловно +удалять S3-объект, поскольку метаданные могли успешно сохраниться. + +Reconciler (src.reconcile) по умолчанию dry-run. --apply удаляет только объекты без +метаданных, старше grace (по умолчанию 600 с, минимум 60 с при deadline запроса 30 с). +Под тем же lock он заново проверяет и наличие метаданных, и LastModified объекта. +Существующие метаданные защищают объект независимо от ссылки из сообщения; свежие +и занятые ключи пропускаются. Это исключает удаление действующей загрузки при обычном +продвижении процессов; произвольная многоминутная приостановка OS не является SLA. + +Если байты пропали, метаданные остаются, storage_state становится missing, API отвечает +503 вместо ложной готовности. После восстановления из резервной копии reconciler +проверяет реальный размер и SHA-256 и возвращает ready. Он не способен изобрести +потерянные байты. Повторный --apply не создаёт новых эффектов для исправленных записей. +Обход S3 постраничный, метаданных — keyset блоками по 100; сетевые ошибки не считаются +доказательством отсутствующего объекта. Только S3 404/NoSuchKey означает missing. + +Message хранит attachment_ids, а message_attachments обеспечивает внешние ключи. +Все вложения проверяются на принадлежность текущему чату до атомарной записи сообщения. +В хеш идемпотентности входит список вложений; отсутствие списка и [] канонически равны. + +Отдельный процесс src.reconcile_loop выполняет apply каждые 60 секунд с grace 600. +Readiness читает байты зарезервированного S3 control object: metadata LIST не +доказывает готовность volume server после перезапуска. Контрольный объект исключён +из orphan cleanup. Decoder/PUT при отмене запроса завершаются перед закрытием +временного файла и освобождением semaphore, поскольку asyncio не останавливает поток. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..558e5171 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +# Google public mirror of the official Docker Python image; immutable digest. +FROM mirror.gcr.io/library/python@sha256:34386ef0cb081344d7ec1c103ba398e6e9f64e9ab3a1509accc92a4e24a07258 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONUTF8=1 \ + APP_HOST=0.0.0.0 APP_PORT=8080 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && \ + groupadd --gid 10001 messenger && useradd --uid 10001 --gid 10001 --no-create-home messenger +COPY src ./src +COPY migrations ./migrations +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=6s --start-period=60s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=5)" +CMD ["python", "-m", "src"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..2d5b16fa 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,185 @@ -# Отчёт — лабораторная 4 +# Лабораторная 4 — приватное S3 и согласованность -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Парамонов Глеб, БВТ2301. Целевая оценка: **5**, трек 2 «Согласованность двух хранилищ». +Ветка: `solution/bvt2301-paramonov`. Основа — собственная реализация ЛР3. -## Запуск из чистого clone +## Воспроизводимый запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine и Compose v2, Python 3.12+ для управляющих скриптов. +На Windows используется Docker Desktop/WSL2; команды Python запускать с `-X utf8`. -## Архитектура и решения +```sh +python -X utf8 scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build --wait +python -m venv .venv +# Linux: .venv/bin/python; Windows: .venv/Scripts/python.exe +.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-test.txt +.venv/bin/python -X utf8 scripts/check_contract.py +.venv/bin/python -X utf8 scripts/verify_stack.py +.venv/bin/python -X utf8 scripts/reconciliation_experiment.py +docker compose down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`bootstrap.py` создаёт локальные `.env`, `.env.api`, `.env.storage-admin` и +`secrets/s3.json`. Значения случайны, не выводятся, исключены из Git и build context. +Повтор сохраняет существующую конфигурацию, неполную конфигурацию не перезаписывает. +API получает только учётные данные для `originals/*` в `messenger`, storage-init — +отдельные административные. PostgreSQL и bucket создаются автоматически; миграции +проверяют checksum и сериализуются advisory lock. `storage-init` допускает повтор. +`down` сохраняет volumes; `docker compose down -v` намеренно удаляет учебную БД и +файлы, применять только при осознанном полном сбросе стенда. -## Доказательства критериев +Все образы закреплены digest в Compose/Dockerfile: Python 3.12.15, PostgreSQL 17.11, +Valkey 8.1, HAProxy 3.0, SeaweedFS 4.48. Используется публичное зеркало официальных +образов `mirror.gcr.io`, поскольку Docker Hub недоступен из среды выполнения. +Python-пакеты закреплены в `requirements.txt`. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Архитектура и границы сохранности -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Клиент → HAProxy → 2 API → PostgreSQL (ACL, метаданные, сообщения) и SeaweedFS S3 +(неизменённые оригиналы). Valkey ускоряет профиль, действительность сессии всегда +проверяется в PostgreSQL. Общая транзакционная идемпотентность сообщений на 24 часа, +keyset pagination и ограничения ЛР3 сохранены. Отсутствующий `attachment_ids` и `[]` +имеют одинаковый хеш запроса. Каждое вложение принадлежит одному чату; внешний +пользователь получает 404 и для metadata, и для bytes. Object key назначает сервер. -## Эксперименты и отказы +Перед чтением тела проверяются сессия/ACL/filename/MIME. Тело потоково записывается +во временный файл до 10 MiB; Pillow проверяет формат, целостность и лимит 40M pixels +до декодирования. JPEG дополнительно полностью декодируется: одного `verify()` +недостаточно для обнаружения обрезанного entropy stream. SHA-256 вычисляется по +исходным байтам; авторизованное скачивание идёт через API порциями 64 KiB. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +API удерживает транзакционный advisory lock object key, выполняет PUT, затем INSERT +метаданных и COMMIT. **201 возвращается только после обоих сохранений.** Нет общей +транзакции PostgreSQL/S3: ошибка COMMIT может оставить orphan. Нельзя безусловно +удалять объект при неоднозначном COMMIT — запись могла сохраниться. Reconciler +позже перепроверяет БД под тем же lock и удаляет только доказанный старый orphan. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Отдельный `reconciler` запускает apply каждые 60 секунд с grace 600 секунд. +Одноразовая команда по умолчанию **dry-run**: -## Ограничения и следующие шаги +```sh +docker compose exec api python -m src.reconcile +docker compose exec api python -m src.reconcile --apply +``` -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Порог нельзя уменьшить ниже 60 секунд. Свежие объекты пропускаются; при занятом lock +объект пропускается. Перед удалением снова читаются metadata и HEAD/LastModified. +Любой объект с metadata сохраняется, в том числе ещё не прикреплённый к сообщению. +Обход metadata — keyset страницами по 100; S3 — paginator. Missing/неверный размер +или checksum metadata переводит запись в карантин `missing` (API 503), сохраняя +ссылки. Восстановление из backup требует совпадения настоящих байтов SHA-256/размера, +после чего reconciler возвращает ready. 403/timeout S3 не трактуется как отсутствие. +Потерянные байты без backup автоматически восстановить невозможно. + +## Ограничение ресурсов и приватность + +Каждый API: 0.5 CPU, 512 MiB, tmpfs 32 MiB, не root, read-only rootfs, drop ALL caps. +На реплику одновременно 2 uploads, 1 decoder, 4 downloads, всего 32 запроса; при +занятых upload slots — явный 503, автоматических повторов upload на сервере нет. +Блокирующий decoder/PUT при отмене завершается до освобождения permit/закрытия файла: +отмена coroutine не умеет останавливать Python thread. Socket timeout S3 2/4 секунды, +одна попытка; HTTP deadline 30 секунд, LB 32/35, graceful shutdown API 32/35 секунд. +Отмена может дождаться ограниченной по входным данным операции; это не hard real-time. +PostgreSQL 1 CPU/256 MiB, S3 1 CPU/512 MiB, reconciler 0.25 CPU/192 MiB. + +На хост опубликован только LB `127.0.0.1:8080`. S3 доступен только внутри bridge, +master/filer/volume SeaweedFS привязаны к loopback внутри контейнера, административной +консоли снаружи нет. Anonymous identity отсутствует, public bucket policy запрещена +bootstrap. Readiness зависит от PG и S3; cache необязателен. Это консервативный выбор: +при отказе S3 LB может временно убрать API и для текстовых запросов. + +Готовность S3 проверяется чтением постоянного контрольного объекта +`originals/.storage-health`. Один LIST/HEAD недостаточен: filer metadata может +стать доступен до регистрации volume server после restart. Этот зарезервированный +служебный объект исключён из orphan cleanup. Внутренний LevelDB filer и master state +проверены в `/data/filerldb2` и `/data/m9333`, на том же постоянном S3 volume. + +## Проверки и исследование + +Прогоны выполнены 16.10.2026 на реальных контейнерах. Основной набор: +`python -X utf8 scripts/verify_stack.py`; исследование: +`python -X utf8 scripts/reconciliation_experiment.py`. + +| Критерий | Наблюдение | Доказательство | +|---|---|---| +| Минимум ЛР3, файлы и ACL | public smoke PASS; чужой metadata/content/upload — 404; ссылки другого чата/несуществующие — 404 | evidence/integration.json | +| Границы upload | размер/пиксели 413; повреждение/MIME 415; path/control filename 400 | negative_uploads в integration.json | +| Отключение загрузившей реплики | получены те же bytes с другим X-Instance-Id | upload_then_stop_download | +| Приватность и минимальные права | anonymous GET, создание чужого bucket, запись вне prefix — 403 | private_bucket | +| Прерывание upload | после закрытия сокета число metadata не изменилось | interrupted_upload | +| Отказ S3 | 503; после восстановления bootstrap повторяем | s3_down_status, bootstrap_repeat | +| Restart всего стенда | public persistence PASS; новые StartedAt API/PG/cache/LB/S3/reconciler, volumes сохранены | evidence/restart-processes.json | +| Failover | 97 успешных записей, 2 временные неудачи, восстановление за 4.390 с от начала hook; подтверждения/bytes сохранены | public_failover, evidence/stop-one.json | +| 10 одновременных uploads | все завершились, SHA-256 совпали; лимиты явно отказали в лишних попытках | concurrent_uploads | +| PUT прошёл, DB INSERT упал | PG trigger SQLSTATE 40001: HTTP 503, 0 metadata, 1 новый orphan | evidence/reconciliation.json | +| Missing и безопасное восстановление | dry-run без изменения; apply → missing/503; восстановление bytes + проверка SHA → ready | missing_dry_run, missing_quarantined, restored_from_fixture | +| Grace, используемые файлы и повтор | свежий orphan сохранён; старый удалён; используемый файл сохранён; повтор apply без действий | aged_orphan_*, repeat_noop | +| Отмена blocking обработки | файл не закрывается и permit не освобождается до завершения thread | python -m unittest discover -s checks -v | + +В нагрузке использованы 10 PNG по **8 104 028 байт**, barrier запускает 10 клиентов +одновременно. Все 10 приняты за 3.839–5.971 с с учётом ожидания/повторов; 55 HTTP +попыток: 10 ответов 201 и 45 явных 503. Повтор после 503 выполняет только тестовый +клиент с ограниченным числом попыток; это не exactly-once гарантия upload. +VmHWM двух API после нагрузки: **143744 и 132828 KiB (140.38 и 129.71 MiB)**, +до нагрузки 128148 и 119040 KiB. Это максимум RSS процесса за его жизнь, не чистая +дельта одной операции и не весь container cache/tmpfs. Обновлённый сценарий также +сохраняет cgroup memory.peak/events для чистого проверочного прогона. Один небольшой +прогон не доказывает отсутствие всех возможных OOM при любом файле; границы admission, +bytes/pixels, decoder и container memory действуют независимо от этого измерения. + +Гипотеза: lock + grace + повторная сверка устраняют orphan после разрыва PUT/COMMIT, +не удаляя свежие/используемые файлы; missing можно безопасно скрыть и вернуть после +проверенного восстановления. Эксперимент создаёт собственные синтетические данные, +одноразовый PG trigger только для уникального filename, 200 реальных пар S3/metadata, +удаляет один тестовый объект, сравнивает dry-run/apply/repeat и ждёт настоящий grace. +Нагрузка: 10 одновременных HTTP uploads примерно 8.1 MB PNG с фиксированными границами +повторов только на явный 503; проверяются все checksum и VmHWM каждой API-реплики. + +Результат исследования: обход **221 metadata / 223–224 S3 objects** занимал +**1.431–2.530 с** в последовательных dry-run/apply/restore/repeat проходах. +В эксперименте реальный интервал от создания orphan до итоговой проверки — 90.562 с +при grace 60 с. Старый orphan удалён, новый orphan и используемые файлы сохранены; +после исправлений repeat за 1.989 с не предложил действий. Также обнаружен и удалён +orphan от прерванного предварительного запуска: Windows ограничил длину командной +строки с fixture. Сценарий исправлен на передачу кода через stdin; объекты не были +скрыты/обнулены для получения результата. Данные в evidence/reconciliation.json. +Стоимость примерно линейна числу объектов/метаданных; приведён диапазон реальных +проходов с разными действиями, а не статистически независимый benchmark. На большом +bucket понадобится разбиение по prefix и контроль частоты; grace и locks сохраняются. + +## Чистая копия и исправления первого запуска + +Повторный `git clone --local` в отдельный каталог с новыми PG/S3 volumes и secrets: +build/bootstrap/migrations/ready и полный `verify_stack.py` прошли. Доказательства: +`evidence/clean-clone-integration.json`, `evidence/clean-clone-restart.json`. +Дополнительно проверены валидный JPEG побайтно и обрезанный JPEG → 415. +Failover: 115 успешных записей, 2 временные ошибки, восстановление 4.328 с. +Все 10 загрузок завершились; cgroup v1 peak **151740416 / 162308096 байт** +(144.71 / 154.79 MiB), `failcnt=0`, `oom_kill=0` на обеих репликах. + +На пустом S3 первый PUT может ждать создания volumes: bootstrap теперь ограниченно +повторяет операцию с фиксированным служебным ключом и проверяет чтение bytes. +Первый запуск выявил также необходимость штатного nonroot uid 1000 для S3 при +drop ALL caps и проблему параллельной сборки одинакового тега у нескольких сервисов: +`build` объявлен только у API, остальные используют общий готовый образ. +Сбор памяти поддерживает cgroup v1 и v2, поскольку этот Docker/WSL использует v1. + +## Ограничения и источники + +Локальный компьютер: Windows 10, i5-11400H, 8 GB RAM; WSL2 3 GB RAM/4 CPU/4 GB swap. +Одиночные PG/S3/LB и один host остаются SPOF. Persistent volume переживает restart, +но не потерю диска. Listing/HEAD имеют линейную стоимость; результаты локального +стенда не являются SLA. Retention не удаляет пользовательские вложения автоматически. + +При реализации, запуске, исследовании и подготовке отчёта использован OpenAI Codex. +Публичные контракты и tests преподавателя не изменялись. Основные источники: +[SeaweedFS](https://github.com/seaweedfs/seaweedfs), +[S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), +[Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html), +[PostgreSQL advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS). + +## Автоматическая проверка приложения + +`.github/workflows/application.yml` добавляет отдельный job: проверка контрактов, bootstrap (когда нужен), сборка, ожидание readiness и public smoke. Он предназначен для изолированного одноразового Actions-compatible Linux runner с Docker Compose v2; secrets создаются внутри job, после проверки удаляются только его Compose volumes. На GitFlic/Gitea запуск зависит от настроек платформы и runner преподавателя. Успешный удалённый CI этим отчётом не заявляется: приведённые доказательства получены локально, включая чистый clone. Полные fault/load эксперименты воспроизводятся отдельными командами отчёта. diff --git a/checks/test_offload.py b/checks/test_offload.py new file mode 100644 index 00000000..122821af --- /dev/null +++ b/checks/test_offload.py @@ -0,0 +1,44 @@ +"""A cancelled upload must not release memory/file leases while its thread runs.""" +import asyncio +import tempfile +import threading +import unittest + +from src.offload import complete_thread + + +class CancellationTest(unittest.IsolatedAsyncioTestCase): + async def test_file_and_permit_live_until_worker_finishes(self): + entered, finish = threading.Event(), threading.Event() + semaphore = asyncio.Semaphore(1) + observed = [] + + def blocking(file): + entered.set() + if not finish.wait(3): + raise AssertionError('Test failed to release blocking worker') + observed.append(file.closed) + file.write(b'still owned') + + async def request(): + async with semaphore: + with tempfile.TemporaryFile() as file: + await complete_thread(blocking, file) + + request_task = asyncio.create_task(request()) + try: + self.assertTrue(await asyncio.to_thread(entered.wait, 3)) + request_task.cancel() + await asyncio.sleep(.02) + self.assertTrue(semaphore.locked()) + self.assertFalse(request_task.done()) + finally: + finish.set() + with self.assertRaises(asyncio.CancelledError): + await request_task + self.assertEqual(observed, [False]) + self.assertFalse(semaphore.locked()) + + +if __name__ == '__main__': + unittest.main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..2936d662 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,102 @@ +x-app: &app + image: mtusi-messenger:lab4 + env_file: .env.api + cpus: 1.0 + mem_limit: 256m + pids_limit: 64 + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 35s + logging: + driver: json-file + options: {max-size: 5m, max-file: "2"} +services: + db: + image: mirror.gcr.io/library/postgres@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/bootstrap.py} + command: [postgres, -c, shared_buffers=64MB, -c, max_connections=30] + volumes: [pgdata:/var/lib/postgresql/data] + cpus: 1.0 + mem_limit: 256m + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + cache: + image: mirror.gcr.io/valkey/valkey@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: [valkey-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + cpus: 0.25 + mem_limit: 64m + healthcheck: + test: [CMD, valkey-cli, ping] + interval: 2s + timeout: 1s + retries: 20 + migrate: + <<: *app + command: [python, -m, src.migrate] + healthcheck: {disable: true} + depends_on: + db: {condition: service_healthy} + api: + <<: *app + build: . + deploy: + replicas: 2 + cpus: 0.5 + mem_limit: 512m + tmpfs: ["/tmp:size=32m,mode=1777"] + depends_on: + migrate: {condition: service_completed_successfully} + cache: {condition: service_healthy} + storage-init: {condition: service_completed_successfully} + s3: + image: mirror.gcr.io/chrislusf/seaweedfs@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d + user: "1000:1000" + command: [server, -dir=/data, -ip=127.0.0.1, -ip.bind=127.0.0.1, -s3, -s3.ip.bind=0.0.0.0, -s3.config=/etc/seaweedfs/s3.json, -s3.port.iceberg=0, -s3.port.lance=0, -s3.autoCreateBucket=false, -volume.max=10, -master.volumeSizeLimitMB=100, -filer.concurrentUploadLimitMB=64] + environment: {GOMEMLIMIT: 384MiB, GOMAXPROCS: "2"} + volumes: ["s3data:/data", "./secrets/s3.json:/etc/seaweedfs/s3.json:ro"] + cpus: 1.0 + mem_limit: 512m + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 20s + storage-init: + <<: *app + env_file: .env.storage-admin + command: [python, -m, src.bootstrap_storage] + healthcheck: {disable: true} + depends_on: [s3] + reconciler: + <<: *app + cpus: 0.25 + mem_limit: 192m + command: [python, -m, src.reconcile_loop] + healthcheck: + test: [CMD, python, -c, "import os; os.kill(1,0)"] + interval: 10s + timeout: 2s + start_period: 5s + retries: 3 + depends_on: + migrate: {condition: service_completed_successfully} + storage-init: {condition: service_completed_successfully} + lb: + image: mirror.gcr.io/library/haproxy@sha256:d84eb36e91830bcaa67b23cfde1909850b99f11eb60da3abd185bd0daf3f75bb + ports: ["127.0.0.1:8080:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + cpus: 0.5 + mem_limit: 64m + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + depends_on: + api: {condition: service_healthy} +volumes: + pgdata: + s3data: diff --git a/evidence/clean-clone-integration.json b/evidence/clean-clone-integration.json new file mode 100644 index 00000000..8eeed9e3 --- /dev/null +++ b/evidence/clean-clone-integration.json @@ -0,0 +1,198 @@ +{ + "public_smoke": "PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.", + "negative_uploads": { + "oversize": 413, + "corrupt": 415, + "truncated": 415, + "mismatch": 415, + "unsupported": 415, + "path": 400, + "backslash": 400, + "control": 400, + "pixels": 413 + }, + "jpeg_validation": "valid JPEG returned byte-for-byte; truncated entropy stream rejected with 415", + "attachment_acl_and_canonical_body": "foreign/unknown attachments 404; absent and [] attachment_ids replay same message", + "private_bucket": { + "anonymous_get": 403, + "api_create_other_bucket": 403, + "api_write_outside_prefix": 403 + }, + "upload_then_stop_download": { + "stopped_container": "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa", + "uploader_instance": "d1629e5d4ce04df4bd4ea3d1ae198213", + "downloader_instance": "da42b3ba785949008f795e135670b728", + "sha256_matches": true + }, + "interrupted_upload": "partial body closed: no attachment metadata", + "s3_down_status": 503, + "bootstrap_repeat": "Private bucket ready; bootstrap is repeatable", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 115, \"failed_attempts\": 2, \"first_recovery_seconds\": 4.328}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "concurrent_uploads": { + "clients": 10, + "bytes_each": 8104028, + "before_memory_kib": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "VmRSS": 94248, + "VmHWM": 119968, + "cgroup": { + "version": 1, + "peak_bytes": 105979904, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "VmRSS": 98060, + "VmHWM": 129428, + "cgroup": { + "version": 1, + "peak_bytes": 108998656, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + } + }, + "after_memory_kib": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "VmRSS": 134000, + "VmHWM": 134192, + "cgroup": { + "version": 1, + "peak_bytes": 151740416, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "VmRSS": 144808, + "VmHWM": 145008, + "cgroup": { + "version": 1, + "peak_bytes": 162308096, + "failcnt": 0, + "oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 0\n" + } + } + }, + "uploads": [ + { + "i": 0, + "seconds": 4.813, + "statuses": [ + 201 + ] + }, + { + "i": 1, + "seconds": 6.912, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 2, + "seconds": 6.376, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 3, + "seconds": 7.03, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 4, + "seconds": 6.32, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 5, + "seconds": 4.5, + "statuses": [ + 201 + ] + }, + { + "i": 6, + "seconds": 6.258, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 7, + "seconds": 4.847, + "statuses": [ + 201 + ] + }, + { + "i": 8, + "seconds": 4.534, + "statuses": [ + 201 + ] + }, + { + "i": 9, + "seconds": 6.628, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + } + ] + }, + "passed": true +} diff --git a/evidence/clean-clone-restart.json b/evidence/clean-clone-restart.json new file mode 100644 index 00000000..27b8d0a1 --- /dev/null +++ b/evidence/clean-clone-restart.json @@ -0,0 +1,591 @@ +{ + "before": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 89629, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:04:51.580983558Z", + "FinishedAt": "2026-10-16T19:04:50.691195534Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:20.211235074Z", + "End": "2026-10-16T19:05:20.652084583Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:25.652679198Z", + "End": "2026-10-16T19:05:26.123505815Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:31.126069514Z", + "End": "2026-10-16T19:05:31.609616392Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:37.925954313Z", + "End": "2026-10-16T19:05:38.361362305Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:43.362382377Z", + "End": "2026-10-16T19:05:43.839170887Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57715, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:40.202839878Z", + "FinishedAt": "2026-10-16T14:21:39.839302297Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:21.557583747Z", + "End": "2026-10-16T19:05:22.142090738Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:27.143338283Z", + "End": "2026-10-16T19:05:27.622167618Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:32.623688976Z", + "End": "2026-10-16T19:05:33.046515405Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:39.363083138Z", + "End": "2026-10-16T19:05:39.842740067Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:44.84414073Z", + "End": "2026-10-16T19:05:45.339352229Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "2441f99f958bbd516d0ab7b93eb96e09b7f875a4164f9ff9d645a9aede2dc6c5": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57338, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:37.770464288Z", + "FinishedAt": "2026-10-16T14:21:37.196628506Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:38.745087502Z", + "End": "2026-10-16T19:05:38.789808433Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:40.790459849Z", + "End": "2026-10-16T19:05:40.835150776Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:42.836652693Z", + "End": "2026-10-16T19:05:42.881695222Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:44.882830897Z", + "End": "2026-10-16T19:05:44.932299003Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:05:46.933638229Z", + "End": "2026-10-16T19:05:46.99111665Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "e4fb86e6ec85e6c96ae794ba39e8fc93d179bada127bfb8cebe4f5a5361877d1": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57302, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:37.642224221Z", + "FinishedAt": "2026-10-16T14:21:37.158949295Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:39.218402845Z", + "End": "2026-10-16T19:05:39.23869686Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:41.239357605Z", + "End": "2026-10-16T19:05:41.259293587Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:43.261038537Z", + "End": "2026-10-16T19:05:43.287970539Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:45.28872912Z", + "End": "2026-10-16T19:05:45.310829129Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:05:47.311317596Z", + "End": "2026-10-16T19:05:47.331220589Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "dad1c78ccfbd99a5f2e5ee467c3216b81bb70dd4a9d9bfa957a1514050e762bc": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57396, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:37.890968447Z", + "FinishedAt": "2026-10-16T14:21:37.097695286Z" + } + }, + "6e8f2c3c72100a9e08ad6bd157697931fcb23b7846c5929a86033ee1e5f5a978": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 90601, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:21.285471656Z", + "FinishedAt": "2026-10-16T19:05:20.730580171Z" + } + }, + "572e906935d7ec6f3dd28d51caaee7d25696558344f288631ba92977b5f3bc45": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 57604, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:21:39.595976344Z", + "FinishedAt": "2026-10-16T14:21:39.251848171Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:04:58.650013908Z", + "End": "2026-10-16T19:04:58.687567121Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:10.130199277Z", + "End": "2026-10-16T19:05:10.161910968Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:20.162424877Z", + "End": "2026-10-16T19:05:20.192425192Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:30.193378293Z", + "End": "2026-10-16T19:05:30.223733363Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:41.540431923Z", + "End": "2026-10-16T19:05:41.575560046Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + } + }, + "after": { + "b0a8d4d6c153c378aadf6303f33846b7fab572564fc36920eb6a1c3e1bed62aa": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91908, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:51.503236355Z", + "FinishedAt": "2026-10-16T19:05:51.051892302Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:43.362382377Z", + "End": "2026-10-16T19:05:43.839170887Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:48.839798292Z", + "End": "2026-10-16T19:05:49.453801773Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:56.761820425Z", + "End": "2026-10-16T19:05:57.223778281Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:06:02.22498739Z", + "End": "2026-10-16T19:06:02.671587739Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:06:09.099465314Z", + "End": "2026-10-16T19:06:09.563981298Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "395705f22be6d83f694cc98e96fe5bfe628151af93fd5f1e939704ff21fc5dc8": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 92149, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:53.745039633Z", + "FinishedAt": "2026-10-16T19:05:53.471949599Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:39.363083138Z", + "End": "2026-10-16T19:05:39.842740067Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:05:44.84414073Z", + "End": "2026-10-16T19:05:45.339352229Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:50.339801368Z", + "End": "2026-10-16T19:05:52.041595819Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1357, in do_open\n h.request(req.get_method(), req.selector, req.data, headers,\n File \"/usr/local/lib/python3.12/http/client.py\", line 1386, in request\n self._send_request(method, url, body, headers, encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1432, in _send_request\n self.endheaders(body, encode_chunked=encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1381, in endheaders\n self._send_output(message_body, encode_chunked=encode_chunked)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1141, in _send_output\n self.send(msg)\n File \"/usr/local/lib/python3.12/http/client.py\", line 1085, in send\n self.connect()\n File \"/usr/local/lib/python3.12/http/client.py\", line 1051, in connect\n self.sock = self._create_connection(\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/socket.py\", line 865, in create_connection\n raise exceptions[0]\n File \"/usr/local/lib/python3.12/socket.py\", line 850, in create_connection\n sock.connect(sa)\nConnectionRefusedError: [Errno 111] Connection refused\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 515, in open\n response = self._open(req, data)\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 532, in _open\n result = self._call_chain(self.handle_open, protocol, protocol +\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1386, in http_open\n return self.do_open(http.client.HTTPConnection, req)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 1360, in do_open\n raise URLError(err)\nurllib.error.URLError: \n" + }, + { + "Start": "2026-10-16T19:05:58.87057203Z", + "End": "2026-10-16T19:05:59.550549126Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T19:06:05.214084373Z", + "End": "2026-10-16T19:06:06.387122841Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "2441f99f958bbd516d0ab7b93eb96e09b7f875a4164f9ff9d645a9aede2dc6c5": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91717, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:50.585367281Z", + "FinishedAt": "2026-10-16T19:05:50.114533944Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:06:01.764202467Z", + "End": "2026-10-16T19:06:01.803981152Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:03.805033747Z", + "End": "2026-10-16T19:06:03.844587109Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:07.273151038Z", + "End": "2026-10-16T19:06:07.316174892Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:09.316906087Z", + "End": "2026-10-16T19:06:09.357124824Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T19:06:11.358455466Z", + "End": "2026-10-16T19:06:11.402615578Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "e4fb86e6ec85e6c96ae794ba39e8fc93d179bada127bfb8cebe4f5a5361877d1": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91752, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:50.786619159Z", + "FinishedAt": "2026-10-16T19:05:50.084558012Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:06:01.623734818Z", + "End": "2026-10-16T19:06:01.646386929Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:03.646921834Z", + "End": "2026-10-16T19:06:03.668289714Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:07.09579857Z", + "End": "2026-10-16T19:06:07.114392094Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:09.115447984Z", + "End": "2026-10-16T19:06:09.135323666Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T19:06:11.13641835Z", + "End": "2026-10-16T19:06:11.15433417Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "dad1c78ccfbd99a5f2e5ee467c3216b81bb70dd4a9d9bfa957a1514050e762bc": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 91827, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:51.121747424Z", + "FinishedAt": "2026-10-16T19:05:50.233763703Z" + } + }, + "6e8f2c3c72100a9e08ad6bd157697931fcb23b7846c5929a86033ee1e5f5a978": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 92676, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:06:11.831976196Z", + "FinishedAt": "2026-10-16T19:06:11.505939192Z" + } + }, + "572e906935d7ec6f3dd28d51caaee7d25696558344f288631ba92977b5f3bc45": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 92038, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T19:05:52.824086086Z", + "FinishedAt": "2026-10-16T19:05:52.546565655Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T19:05:30.193378293Z", + "End": "2026-10-16T19:05:30.223733363Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:41.540431923Z", + "End": "2026-10-16T19:05:41.575560046Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:51.576781033Z", + "End": "2026-10-16T19:05:51.823968089Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:05:57.973400351Z", + "End": "2026-10-16T19:05:58.003734353Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T19:06:09.432278398Z", + "End": "2026-10-16T19:06:09.46125484Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + } + }, + "volumes_removed": false +} diff --git a/evidence/integration.json b/evidence/integration.json new file mode 100644 index 00000000..e7e5fa8a --- /dev/null +++ b/evidence/integration.json @@ -0,0 +1,173 @@ +{ + "public_smoke": "PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.", + "negative_uploads": { + "oversize": 413, + "corrupt": 415, + "truncated": 415, + "mismatch": 415, + "unsupported": 415, + "path": 400, + "backslash": 400, + "control": 400, + "pixels": 413 + }, + "attachment_acl_and_canonical_body": "foreign/unknown attachments 404; absent and [] attachment_ids replay same message", + "private_bucket": { + "anonymous_get": 403, + "api_create_other_bucket": 403, + "api_write_outside_prefix": 403 + }, + "upload_then_stop_download": { + "stopped_container": "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62", + "uploader_instance": "f47f01494e104a8cb695f06e3fb350cb", + "downloader_instance": "7f5408689e7e4d9ea143d3308938f777", + "sha256_matches": true + }, + "interrupted_upload": "partial body closed: no attachment metadata", + "s3_down_status": 503, + "bootstrap_repeat": "Private bucket ready; bootstrap is repeatable", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 97, \"failed_attempts\": 2, \"first_recovery_seconds\": 4.39}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "concurrent_uploads": { + "clients": 10, + "bytes_each": 8104028, + "before_memory_kib": { + "346658d6ea74c68d5f3286701db46add73316e3766e62a104dccf47688871165": { + "VmRSS": 96168, + "VmHWM": 128148 + }, + "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62": { + "VmRSS": 93304, + "VmHWM": 119040 + } + }, + "after_memory_kib": { + "346658d6ea74c68d5f3286701db46add73316e3766e62a104dccf47688871165": { + "VmRSS": 143676, + "VmHWM": 143744 + }, + "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62": { + "VmRSS": 132760, + "VmHWM": 132828 + } + }, + "uploads": [ + { + "i": 0, + "seconds": 5.971, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 1, + "seconds": 3.902, + "statuses": [ + 201 + ] + }, + { + "i": 2, + "seconds": 3.925, + "statuses": [ + 201 + ] + }, + { + "i": 3, + "seconds": 5.541, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 4, + "seconds": 5.485, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 5, + "seconds": 5.879, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 6, + "seconds": 5.305, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 7, + "seconds": 5.386, + "statuses": [ + 503, + 503, + 503, + 503, + 503, + 503, + 201 + ] + }, + { + "i": 8, + "seconds": 3.839, + "statuses": [ + 201 + ] + }, + { + "i": 9, + "seconds": 3.899, + "statuses": [ + 201 + ] + } + ] + }, + "passed": true +} diff --git a/evidence/reconciliation.json b/evidence/reconciliation.json new file mode 100644 index 00000000..a9cb347d --- /dev/null +++ b/evidence/reconciliation.json @@ -0,0 +1,114 @@ +{ + "put_before_database_failure": { + "http_status": 503, + "orphan": "originals/e727289d-3f10-4c0a-80fe-d67fa0d3e39b", + "metadata_rows": 0 + }, + "fresh_orphan_preserved": { + "apply": true, + "grace_seconds": 60, + "objects_scanned": 24, + "metadata_scanned": 21, + "fresh_skipped": 6, + "busy_skipped": 0, + "actions": [], + "elapsed_seconds": 0.227435 + }, + "dataset": "Created 200 synthetic object/metadata pairs", + "missing_dry_run": { + "apply": false, + "grace_seconds": 60, + "objects_scanned": 223, + "metadata_scanned": 221, + "fresh_skipped": 202, + "busy_skipped": 0, + "actions": [ + { + "action": "delete_orphan", + "key": "originals/9b7c845c-ec7b-4bb5-b7e9-8b0ce004a8a1" + }, + { + "action": "quarantine_missing", + "id": "456ebe3c-307c-4b09-8738-51bf109d6645" + } + ], + "elapsed_seconds": 2.334789 + }, + "missing_quarantined": { + "apply": true, + "grace_seconds": 60, + "objects_scanned": 223, + "metadata_scanned": 221, + "fresh_skipped": 202, + "busy_skipped": 0, + "actions": [ + { + "action": "delete_orphan", + "key": "originals/9b7c845c-ec7b-4bb5-b7e9-8b0ce004a8a1" + }, + { + "action": "quarantine_missing", + "id": "456ebe3c-307c-4b09-8738-51bf109d6645" + } + ], + "elapsed_seconds": 2.045356 + }, + "restored_from_fixture": { + "apply": true, + "grace_seconds": 60, + "objects_scanned": 223, + "metadata_scanned": 221, + "fresh_skipped": 203, + "busy_skipped": 0, + "actions": [ + { + "action": "restore_ready", + "id": "456ebe3c-307c-4b09-8738-51bf109d6645" + } + ], + "elapsed_seconds": 1.81951 + }, + "aged_orphan_dry_run": { + "apply": false, + "grace_seconds": 60, + "objects_scanned": 224, + "metadata_scanned": 221, + "fresh_skipped": 2, + "busy_skipped": 0, + "actions": [ + { + "action": "delete_orphan", + "key": "originals/e727289d-3f10-4c0a-80fe-d67fa0d3e39b" + } + ], + "elapsed_seconds": 1.43061 + }, + "aged_orphan_apply": { + "apply": true, + "grace_seconds": 60, + "objects_scanned": 224, + "metadata_scanned": 221, + "fresh_skipped": 2, + "busy_skipped": 0, + "actions": [ + { + "action": "delete_orphan", + "key": "originals/e727289d-3f10-4c0a-80fe-d67fa0d3e39b" + } + ], + "elapsed_seconds": 2.529683 + }, + "repeat_noop": { + "apply": true, + "grace_seconds": 60, + "objects_scanned": 223, + "metadata_scanned": 221, + "fresh_skipped": 2, + "busy_skipped": 0, + "actions": [], + "elapsed_seconds": 1.988552 + }, + "used_and_fresh_preserved": true, + "actual_grace_elapsed_seconds": 90.562, + "passed": true +} diff --git a/evidence/restart-processes.json b/evidence/restart-processes.json new file mode 100644 index 00000000..2b04b225 --- /dev/null +++ b/evidence/restart-processes.json @@ -0,0 +1,519 @@ +{ + "before": { + "346658d6ea74c68d5f3286701db46add73316e3766e62a104dccf47688871165": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 36581, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:08:21.580671345Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:09:24.539239881Z", + "End": "2026-10-16T14:09:25.151496968Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:30.152780464Z", + "End": "2026-10-16T14:09:30.828131635Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:35.829816852Z", + "End": "2026-10-16T14:09:36.408127285Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:41.409748596Z", + "End": "2026-10-16T14:09:42.14110257Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T14:09:47.142902171Z", + "End": "2026-10-16T14:09:48.152735271Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 37800, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:08:53.330660301Z", + "FinishedAt": "2026-10-16T14:08:52.503750476Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:09:21.320428185Z", + "End": "2026-10-16T14:09:21.922559653Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:26.924424461Z", + "End": "2026-10-16T14:09:27.524476187Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:32.525560339Z", + "End": "2026-10-16T14:09:33.044744291Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:38.045504543Z", + "End": "2026-10-16T14:09:38.654444122Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:09:43.656141906Z", + "End": "2026-10-16T14:09:44.246486125Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "8f656628fbfbd15e7bdebc41a4ca9472fe7f8672ff6f59a6077387951e05948d": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 32882, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:06:24.580942977Z", + "FinishedAt": "2026-10-16T14:06:23.730651894Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:09:39.168440266Z", + "End": "2026-10-16T14:09:39.21813942Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:09:41.219958037Z", + "End": "2026-10-16T14:09:41.27147249Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:09:43.27276482Z", + "End": "2026-10-16T14:09:43.338549433Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:09:45.339720406Z", + "End": "2026-10-16T14:09:45.417240746Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:09:47.418480717Z", + "End": "2026-10-16T14:09:47.470069589Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "278c682a81890c7593e820d466f58cda087cdf82a400cfde5e3cd348ad3bd68b": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 32844, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:06:24.38084374Z", + "FinishedAt": "2026-10-16T14:06:23.718777344Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:09:40.339646203Z", + "End": "2026-10-16T14:09:40.364151102Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:09:42.364547455Z", + "End": "2026-10-16T14:09:42.384479581Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:09:44.385296431Z", + "End": "2026-10-16T14:09:44.423578703Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:09:46.424183162Z", + "End": "2026-10-16T14:09:46.449339435Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:09:48.450683641Z", + "End": "2026-10-16T14:09:48.47572017Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "5607ad6e81a9c4d34445859c0ccce0ec80cbbc8c24f1b15e6aea03e43ebb4c04": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 36887, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:08:27.927850062Z", + "FinishedAt": "2026-10-16T14:08:11.973547627Z" + } + }, + "c43bc327744c7cc1791783959bd3b1360682bb2b87474065d74278733e302b00": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 38743, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:22.72732595Z", + "FinishedAt": "2026-10-16T14:09:21.993526243Z" + } + }, + "0c93d589240b0f5b65f97546cb9a268927eb96590d2de44934ccc527f2453b7a": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 36601, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:08:21.589972218Z", + "FinishedAt": "0001-01-01T00:00:00Z" + } + } + }, + "after": { + "346658d6ea74c68d5f3286701db46add73316e3766e62a104dccf47688871165": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 40035, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:52.681710318Z", + "FinishedAt": "2026-10-16T14:09:52.02235909Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:09:41.409748596Z", + "End": "2026-10-16T14:09:42.14110257Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T14:09:47.142902171Z", + "End": "2026-10-16T14:09:48.152735271Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T14:09:58.073831796Z", + "End": "2026-10-16T14:09:58.706791676Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:10:03.707608596Z", + "End": "2026-10-16T14:10:04.492799802Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:10:09.493136226Z", + "End": "2026-10-16T14:10:10.12209133Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62": { + "service": "api", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 40110, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:52.83002087Z", + "FinishedAt": "2026-10-16T14:09:52.108187969Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:09:43.656141906Z", + "End": "2026-10-16T14:09:44.246486125Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T14:09:49.248279953Z", + "End": "2026-10-16T14:09:50.893724452Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-16T14:09:58.169949852Z", + "End": "2026-10-16T14:09:58.817003804Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:10:03.818532986Z", + "End": "2026-10-16T14:10:04.524799447Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + }, + { + "Start": "2026-10-16T14:10:09.52543754Z", + "End": "2026-10-16T14:10:10.123620746Z", + "ExitCode": 1, + "Output": "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n" + } + ] + } + } + }, + "8f656628fbfbd15e7bdebc41a4ca9472fe7f8672ff6f59a6077387951e05948d": { + "service": "db", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 39775, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:51.548282042Z", + "FinishedAt": "2026-10-16T14:09:51.045352054Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:10:02.900569926Z", + "End": "2026-10-16T14:10:02.953170019Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:10:04.954463426Z", + "End": "2026-10-16T14:10:05.017716373Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:10:07.018532825Z", + "End": "2026-10-16T14:10:07.069487806Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:10:09.070896767Z", + "End": "2026-10-16T14:10:09.139568759Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + }, + { + "Start": "2026-10-16T14:10:11.140692794Z", + "End": "2026-10-16T14:10:11.217579429Z", + "ExitCode": 0, + "Output": "/var/run/postgresql:5432 - accepting connections\n" + } + ] + } + } + }, + "278c682a81890c7593e820d466f58cda087cdf82a400cfde5e3cd348ad3bd68b": { + "service": "cache", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 39809, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:51.75050958Z", + "FinishedAt": "2026-10-16T14:09:51.092619567Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-16T14:10:02.992360531Z", + "End": "2026-10-16T14:10:03.025255403Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:10:05.02582096Z", + "End": "2026-10-16T14:10:05.047883105Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:10:07.048190623Z", + "End": "2026-10-16T14:10:07.073697427Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:10:09.07443307Z", + "End": "2026-10-16T14:10:09.109965678Z", + "ExitCode": 0, + "Output": "PONG\n" + }, + { + "Start": "2026-10-16T14:10:11.110407295Z", + "End": "2026-10-16T14:10:11.159880115Z", + "ExitCode": 0, + "Output": "PONG\n" + } + ] + } + } + }, + "5607ad6e81a9c4d34445859c0ccce0ec80cbbc8c24f1b15e6aea03e43ebb4c04": { + "service": "lb", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 39874, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:51.907192628Z", + "FinishedAt": "2026-10-16T14:09:50.984379631Z" + } + }, + "c43bc327744c7cc1791783959bd3b1360682bb2b87474065d74278733e302b00": { + "service": "s3", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 40689, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:10:11.353256258Z", + "FinishedAt": "2026-10-16T14:10:11.012750159Z" + } + }, + "0c93d589240b0f5b65f97546cb9a268927eb96590d2de44934ccc527f2453b7a": { + "service": "reconciler", + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 39960, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-16T14:09:52.306778941Z", + "FinishedAt": "2026-10-16T14:09:51.807513915Z" + } + } + }, + "volumes_removed": false +} diff --git a/evidence/stop-one.json b/evidence/stop-one.json new file mode 100644 index 00000000..392c6644 --- /dev/null +++ b/evidence/stop-one.json @@ -0,0 +1,11 @@ +{ + "target_instance": "001ae5c767bb4979a2a6a080bd8bf505", + "stopped_container": "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62", + "before_containers": [ + "346658d6ea74c68d5f3286701db46add73316e3766e62a104dccf47688871165", + "a6b9b7e0474a809dc487dadc70b769a3fc7b271a47ee9045f7999e7514465a62" + ], + "after_running_containers": [ + "346658d6ea74c68d5f3286701db46add73316e3766e62a104dccf47688871165" + ] +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..48689f30 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,30 @@ +global + log stdout format raw local0 + maxconn 256 +defaults + mode http + log global + timeout connect 1s + timeout client 35s + timeout server 32s + timeout check 1s + timeout queue 2s + retries 0 + option http-server-close +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s +frontend public + bind :8080 + default_backend api +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template replica 1-4 api:8080 check inter 500ms fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr none maxconn 32 diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..3b9cda97 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,35 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX sessions_expiry ON sessions(expires_at); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY(chat_id,user_id) +); +CREATE INDEX members_user_chat ON members(user_id,chat_id); +CREATE INDEX chats_order ON chats(created_at,id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY(chat_id,sender_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX messages_history ON messages(chat_id,created_at,id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..582b0c4f --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY(actor_id,chat_id,key) +); +CREATE INDEX idempotency_expiry ON idempotency(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..839d6753 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,25 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + storage_state text NOT NULL DEFAULT 'ready' CHECK (storage_state IN ('ready','missing')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id), + FOREIGN KEY (chat_id, owner_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX attachments_chat ON attachments(chat_id,created_at,id); +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_content_check + CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0)); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + PRIMARY KEY(message_id,attachment_id) +); diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..9dff8880 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +-r requirements.txt +httpx==0.28.1 +matplotlib==3.10.8 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..cc889419 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +fastapi==0.143.2 +h11==0.16.0 +idna==3.20 +opentelemetry-api==1.45.1 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.11 +pydantic==2.14.1 +pydantic_core==2.50.1 +redis==8.1.0 +starlette==1.8.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +tzdata==2026.5 +uvicorn==0.54.0 +boto3==1.43.114 +Pillow==12.3.0 +botocore==1.43.114 +jmespath==1.1.0 +s3transfer==0.19.2 +python-dateutil==2.9.0.post0 +urllib3==2.8.0 +six==1.17.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..eb615253 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,32 @@ +"""Create separate local DB, API and S3 administration credentials once, without printing them.""" +import json +from pathlib import Path +import secrets + +root=Path(__file__).resolve().parents[1] +target=root/'.env' +if target.exists(): + required=['.env.api','.env.storage-admin','secrets/s3.json'] + if not all((root/p).exists() for p in required): + raise SystemExit('Incomplete bootstrap: preserve existing secrets and restore the missing local files') + print('Existing local configuration preserved') +else: + password=secrets.token_hex(24) + admin_key,admin_secret=secrets.token_hex(12),secrets.token_hex(32) + api_key,api_secret=secrets.token_hex(12),secrets.token_hex(32) + (root/'secrets').mkdir(exist_ok=True) + files={ + '.env':f'POSTGRES_PASSWORD={password}\n', + '.env.api':f'DATABASE_URL=postgresql://messenger:{password}@db:5432/messenger\nREDIS_URL=redis://cache:6379/0\nCURSOR_SECRET={secrets.token_hex(32)}\nSESSION_TTL_SECONDS=3600\nS3_ENDPOINT=http://s3:8333\nS3_BUCKET=messenger\nS3_ACCESS_KEY={api_key}\nS3_SECRET_KEY={api_secret}\n', + '.env.storage-admin':f'S3_ENDPOINT=http://s3:8333\nS3_BUCKET=messenger\nS3_ACCESS_KEY={admin_key}\nS3_SECRET_KEY={admin_secret}\n', + 'secrets/s3.json':json.dumps({'identities':[ + {'name':'bootstrap-admin','credentials':[{'accessKey':admin_key,'secretKey':admin_secret}], + 'actions':['Admin','Read','Write','List','Tagging']}, + {'name':'messenger-api','credentials':[{'accessKey':api_key,'secretKey':api_secret}], + 'actions':['Read:messenger/originals/*','Write:messenger/originals/*','List:messenger']} + ]},indent=2)+'\n'} + for name,content in files.items(): + with (root/name).open('x',encoding='utf-8') as f: f.write(content) + try: (root/name).chmod(0o600) + except OSError: pass + print('Created separate local API and storage-admin configuration; no values displayed') diff --git a/scripts/reconciliation_experiment.py b/scripts/reconciliation_experiment.py new file mode 100644 index 00000000..1391ace9 --- /dev/null +++ b/scripts/reconciliation_experiment.py @@ -0,0 +1,120 @@ +"""Inject a real S3/DB split, missing object, dry-run/apply/repeat and age safety.""" +import base64 +import hashlib +import json +import subprocess +import time +import uuid + +import httpx +from stack import ROOT, docker, ready + +OUT=ROOT/'evidence' +OUT.mkdir(exist_ok=True) +result={} + +def sql(q): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',q) +def inside(code): + return subprocess.check_output(['docker','compose','exec','-T','api','python','-'], + input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip() +def reconcile(apply=False): + args=['compose','exec','-T','api','python','-m','src.reconcile','--grace-seconds','60'] + if apply: args.append('--apply') + return json.loads(docker(*args)) +def objects(): + return set(json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps([o['Key'] for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix='originals/') for o in p.get('Contents',[])]))"))) + +data=(ROOT/'tests/fixtures/sample.png').read_bytes() +encoded=base64.b64encode(data).decode() +digest=hashlib.sha256(data).hexdigest() + +with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c: + ready(c) + name='reconcile_'+uuid.uuid4().hex[:18] + password='reconciliation-test-123' + r=c.post('/api/v1/users',json={'username':name,'display_name':'Reconcile','password':password}); assert r.status_code==201,r.text + uid=r.json()['id'] + r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201 + h={'Authorization':'Bearer '+r.json()['token']} + r=c.post('/api/v1/chats',headers=h,json={'title':'Reconciliation','member_ids':[uid]}); assert r.status_code==201 + chat=r.json()['id'] + def upload(filename): + return c.post(f'/api/v1/chats/{chat}/attachments',headers={**h,'Content-Type':'image/png'},params={'filename':filename},content=data) + used=upload('used.png'); missing=upload('missing.png') + assert used.status_code==missing.status_code==201 + used,missing=used.json(),missing.json() + r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[used['id']]}) + assert r.status_code==201 + used_key=sql("SELECT object_key FROM attachments WHERE id='"+used['id']+"'") + missing_key=sql("SELECT object_key FROM attachments WHERE id='"+missing['id']+"'") + filename='fault_'+uuid.uuid4().hex+'.png' + trigger='fault_'+uuid.uuid4().hex + before=objects() + # Scope is one unique synthetic filename; unrelated requests remain unaffected. + sql(f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='{filename}' THEN RAISE EXCEPTION 'synthetic transient failure' USING ERRCODE='40001'; END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {trigger}();") + try: + failed=upload(filename) + assert failed.status_code==503,(failed.status_code,failed.text) + finally: + sql(f'DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}();') + orphaned=objects()-before + assert len(orphaned)==1,orphaned + orphan=next(iter(orphaned)); orphan_created=time.monotonic() + assert sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"'")=='0' + result['put_before_database_failure']={'http_status':503,'orphan':orphan,'metadata_rows':0} + # Fresh orphan must remain even with --apply. + fresh_run=reconcile(True) + assert orphan in objects() + result['fresh_orphan_preserved']=fresh_run + # Generate 200 real object/metadata pairs for a measurable traversal; no mock S3. + setup_code=f'''import base64,io,uuid,os,psycopg +from src.storage import Storage +s=Storage(); data=base64.b64decode({encoded!r}) +with psycopg.connect(os.environ['DATABASE_URL']) as db: + for n in range(200): + aid=uuid.uuid4(); key='originals/'+str(aid) + s.put(key,io.BytesIO(data),len(data),'image/png',{digest!r}) + db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)',(aid,{chat!r},{uid!r},'scan-fixture.png',key,'image/png',len(data),{digest!r})) +print('Created 200 synthetic object/metadata pairs')''' + result['dataset']=inside(setup_code) + inside("from src.storage import Storage; Storage().delete("+repr(missing_key)+")") + before_objects=objects() + dry=reconcile(False) + assert objects()==before_objects + assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='ready' + assert any(a['action']=='quarantine_missing' and a['id']==missing['id'] for a in dry['actions']) + result['missing_dry_run']=dry + applied=reconcile(True) + assert sql("SELECT storage_state FROM attachments WHERE id='"+missing['id']+"'")=='missing' + assert c.get('/api/v1/attachments/'+missing['id'],headers=h).status_code==503 + assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data + result['missing_quarantined']=applied + # Restore from the known test fixture (in reality from backup), then verify byte checksum before publishing ready. + inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(missing_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")") + result['restored_from_fixture']=reconcile(True) + assert c.get('/api/v1/attachments/'+missing['id']+'/content',headers=h).content==data + # Real elapsed grace, not backdated timestamps or a monkeypatched clock. + while time.monotonic()-orphan_created<65: + time.sleep(min(5,65-(time.monotonic()-orphan_created))) + fresh_key='originals/'+str(uuid.uuid4()) + inside("from src.storage import Storage; import io,base64; data=base64.b64decode("+repr(encoded)+"); Storage().put("+repr(fresh_key)+",io.BytesIO(data),len(data),'image/png',"+repr(digest)+")") + before_objects=objects() + dry=reconcile(False) + assert objects()==before_objects + assert any(a=={'action':'delete_orphan','key':orphan} for a in dry['actions']) + assert not any(a.get('key') in (fresh_key,used_key,missing_key) for a in dry['actions']) + result['aged_orphan_dry_run']=dry + result['aged_orphan_apply']=reconcile(True) + after=objects() + assert orphan not in after and fresh_key in after and used_key in after and missing_key in after + repeated=reconcile(True) + assert repeated['actions']==[],repeated + result['repeat_noop']=repeated + assert c.get('/api/v1/attachments/'+used['id']+'/content',headers=h).content==data + result['used_and_fresh_preserved']=True + result['actual_grace_elapsed_seconds']=round(time.monotonic()-orphan_created,3) + result['passed']=True + # Remove only the known test-only fresh orphan after the assertions; not application data. + inside("from src.storage import Storage; Storage().delete("+repr(fresh_key)+")") +(OUT/'reconciliation.json').write_text(json.dumps(result,indent=2)+'\n',encoding='utf-8') +print(json.dumps(result,indent=2)) diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..981fa4d8 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,13 @@ +"""Restart every running replica and dependency, preserving durable volumes.""" +import json +from stack import ROOT, docker, state + +before = state() +docker('compose', 'restart', 'api', 'db', 'cache', 'lb', 's3', 'reconciler') +after = state() +assert set(before) == set(after) +assert all(before[c]['state']['StartedAt'] != after[c]['state']['StartedAt'] for c in before) +(ROOT / 'evidence').mkdir(exist_ok=True) +(ROOT / 'evidence/restart-processes.json').write_text( + json.dumps({'before': before, 'after': after, 'volumes_removed': False}, indent=2) + '\n', + encoding='utf-8') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..d504e1e1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/restart.py diff --git a/scripts/stack.py b/scripts/stack.py new file mode 100644 index 00000000..061de143 --- /dev/null +++ b/scripts/stack.py @@ -0,0 +1,35 @@ +"""Shared bounded Docker helpers for reproducible, scoped failure experiments.""" +import json +from pathlib import Path +import subprocess +import time +ROOT=Path(__file__).resolve().parents[1] +def docker(*args): + return subprocess.check_output(['docker',*args],cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip() +def replicas(): return docker('compose','ps','-q','api').splitlines() + +def ready_replicas(count=2,timeout=60): + deadline=time.monotonic()+timeout + while time.monotonic()10: invalid('Invalid attachments') + ids=[uuid_value(v) for v in ids] + if len(ids)!=len(set(ids)): invalid('Duplicate attachment') + text=string(body['text'],4000,blank=True) + if not text.strip() and not ids: invalid('Message must have content') + return await store.create_message(user['id'],chat_id,text,key,ids) + + def metadata(row): + result={k:v for k,v in row.items() if k not in ('object_key','storage_state')} + result.update(status='ready',variants=[],error_code=None) + return result + + @api.post('/api/v1/chats/{chat_id}/attachments',status_code=201) + async def upload(chat_id:str,request:Request): + user=await actor(request); chat_id=uuid_value(chat_id) + await store.chat(user['id'],chat_id) + if set(request.query_params)!={'filename'} or len(request.query_params.getlist('filename'))!=1: + invalid('Expected one filename') + filename=string(request.query_params['filename'],255) + if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in filename): invalid('Invalid filename') + mime=request.headers.get('content-type','').strip().lower() + if mime not in ('image/png','image/jpeg'): raise ApiError(415,'unsupported_media_type','Expected PNG/JPEG') + if uploads.locked(): raise ApiError(503,'unavailable','Upload slots busy',{'Retry-After':'1'}) + async with uploads: + with tempfile.TemporaryFile() as file: + size=0 + async for chunk in request.stream(): + size+=len(chunk) + if size>MAX_BYTES: raise ApiError(413,'payload_too_large','Upload exceeds 10 MiB') + file.write(chunk) + async with decoders: + digest=await complete_thread(validate_image,file,mime) + row=await store.save_attachment(user['id'],chat_id,filename,mime,size,digest,file,storage) + return metadata(row) + + @api.get('/api/v1/attachments/{attachment_id}') + async def attachment(attachment_id:str,request:Request): + user=await actor(request) + return metadata(await store.attachment(user['id'],uuid_value(attachment_id))) + + @api.get('/api/v1/attachments/{attachment_id}/content') + async def content(attachment_id:str,request:Request): + user=await actor(request) + row=await store.attachment(user['id'],uuid_value(attachment_id)) + if set(request.query_params)-{'variant'} or len(request.query_params.getlist('variant'))>1: invalid('Invalid query') + if request.query_params.get('variant','original')!='original': raise ApiError(404,'not_found','Variant not found') + if downloads.locked(): raise ApiError(503,'unavailable','Download slots busy') + await downloads.acquire() + try: + obj=await asyncio.to_thread(storage.get,row['object_key']) + except BaseException: + downloads.release() + raise + async def chunks(): + try: + while data:=await asyncio.to_thread(obj['Body'].read,65536): + yield data + finally: + obj['Body'].close() + downloads.release() + return StreamingResponse(chunks(),media_type=row['content_type'],headers={ + 'Content-Length':str(row['size_bytes']),'X-Content-Type-Options':'nosniff', + 'Content-Disposition':'attachment'}) + + api.add_middleware(Boundaries,settings=cfg) + return api + + +app=create_app() diff --git a/src/bootstrap_storage.py b/src/bootstrap_storage.py new file mode 100644 index 00000000..574a1d20 --- /dev/null +++ b/src/bootstrap_storage.py @@ -0,0 +1,42 @@ +"""Idempotent private bucket bootstrap; only this one-shot service receives admin credentials.""" +import time +from botocore.exceptions import BotoCoreError, ClientError +from .storage import Storage, HEALTH_KEY, HEALTH_BYTES + +storage=Storage() +for attempt in range(60): + try: + storage.client.list_buckets() + break + except (BotoCoreError,ClientError): + if attempt==59: raise RuntimeError('Object storage did not become available') from None + time.sleep(1) +try: + storage.client.head_bucket(Bucket=storage.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('404','NoSuchBucket'): raise + storage.client.create_bucket(Bucket=storage.bucket) +# No anonymous identity and no public bucket policy. Do not overwrite any policy +# silently on repeat; reject a public policy if an operator added one accidentally. +try: + policy=storage.client.get_bucket_policy(Bucket=storage.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('NoSuchBucketPolicy','404'): raise +else: + raise RuntimeError('Unexpected bucket policy; inspect it before starting API') +deadline=time.monotonic()+90 +while True: + try: + try: + storage.client.head_object(Bucket=storage.bucket, Key=HEALTH_KEY) + except ClientError as exc: + if exc.response['Error']['Code'] not in ('404','NoSuchKey'): raise + # Fixed, reserved key/content: retry after uncertain PUT is safe. + storage.client.put_object(Bucket=storage.bucket, Key=HEALTH_KEY, Body=HEALTH_BYTES) + storage.ready() + break + except (BotoCoreError,ClientError): + if time.monotonic()>=deadline: + raise RuntimeError('Storage volumes did not become ready within bootstrap deadline') from None + time.sleep(1) +print('Private bucket ready; bootstrap is repeatable') diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..7d4e9e84 --- /dev/null +++ b/src/common.py @@ -0,0 +1,101 @@ +"""Strict contract validation and resource-bound cursors.""" +import base64 +import binascii +import hashlib +import hmac +import json +import re +from datetime import datetime, timezone +from uuid import UUID + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status, self.code, self.message = status, code, message + self.headers = headers or {} + + +def invalid(message='Invalid request'): + raise ApiError(400, 'invalid_request', message) + + +def uuid_value(value): + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}', value + ): + invalid('Invalid UUID') + return str(UUID(value)) + + +def string(value, maximum, *, blank=False): + if not isinstance(value, str) or len(value) > maximum or (not blank and not value.strip()): + invalid('Invalid string') + # JSON can encode unpaired surrogates; they are not Unicode scalar values. + try: + value.encode('utf-8') + except UnicodeError: + invalid('Invalid Unicode') + return value + + +def object_fields(value, required, optional=()): + if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): + invalid('Invalid fields') + return value + + +def timestamp(): + return datetime.now(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def encode_cursor(resource, key, secret): + data = json.dumps([resource, *key], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(data + hmac.digest(secret, data, 'sha256')).decode().rstrip('=') + + +def decode_cursor(value, resource, secret): + try: + if len(value) > 1024 or not re.fullmatch(r'[A-Za-z0-9_-]+', value): + raise ValueError() + raw = base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True) + data, signature = raw[:-32], raw[-32:] + if not hmac.compare_digest(hmac.digest(secret, data, 'sha256'), signature): + raise ValueError() + scope, created, identifier = json.loads(data) + if scope != resource or not isinstance(created, str) or not created.endswith('Z'): + raise ValueError() + datetime.fromisoformat(created.replace('Z', '+00:00')) + UUID(identifier) + return created, identifier + except (ValueError, TypeError, binascii.Error, UnicodeError): + invalid('Invalid cursor') + + +def pagination(request, resource, secret): + if any(len(request.query_params.getlist(k)) != 1 for k in request.query_params): + invalid('Duplicate query parameter') + if set(request.query_params) - {'limit', 'cursor'}: + invalid('Unknown query parameter') + value = request.query_params.get('limit', '50') + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + invalid('Limit must be 1..100') + cursor = request.query_params.get('cursor') + return int(value), decode_cursor(cursor, resource, secret) if cursor is not None else None + + +async def json_body(request): + if request.headers.get('content-type', '').split(';')[0].strip().lower() != 'application/json': + raise ApiError(415, 'unsupported_media_type', 'Expected application/json') + def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate key') + result[key] = value + return result + def no_constant(value): + raise ValueError('non-finite number') + try: + return json.loads((await request.body()).decode('utf-8'), object_pairs_hook=unique_pairs, parse_constant=no_constant) + except (ValueError, UnicodeError, RecursionError): + invalid('Invalid JSON') diff --git a/src/http.py b/src/http.py new file mode 100644 index 00000000..9f0baa14 --- /dev/null +++ b/src/http.py @@ -0,0 +1,95 @@ +import asyncio +import json +import logging +import time +from uuid import uuid4 +from starlette.responses import JSONResponse +from starlette.requests import ClientDisconnect +from .common import ApiError +logger=logging.getLogger("messenger") + +def error_response(exc, request_id): + return JSONResponse({'error': {'code': exc.code, 'message': exc.message, 'request_id': request_id}}, + status_code=exc.status, headers={'X-Request-Id': request_id, **exc.headers}) + + +class Boundaries: + """Admission before body buffering; no unbounded application waiting queue.""" + def __init__(self, app, settings): + self.app, self.cfg = app, settings + self.instance_id = uuid4().hex + self.active, self.tokens, self.updated = 0, float(settings.write_burst), time.monotonic() + + async def __call__(self, scope, receive, send): + if scope['type'] != 'http': + return await self.app(scope, receive, send) + request_id, started, status = uuid4().hex, time.monotonic(), 500 + scope.setdefault('state', {})['request_id'] = request_id + started_response = False + async def tracked_send(message): + nonlocal status, started_response + if message['type'] == 'http.response.start': + status, started_response = message['status'], True + message['headers'] = [(k, v) for k, v in message.get('headers', []) if k.lower() != b'x-request-id'] + [(b'x-request-id', request_id.encode()), (b'x-instance-id', self.instance_id.encode())] + await send(message) + admitted = False + try: + if self.active >= self.cfg.max_inflight: + raise ApiError(503, 'unavailable', 'Too many in-flight requests', {'Retry-After': '1'}) + now = time.monotonic() + self.tokens = min(self.cfg.write_burst, self.tokens + (now - self.updated) * self.cfg.write_rps) + self.updated = now + if scope['method'] in ('POST', 'PUT', 'PATCH'): + if self.tokens < 1: + raise ApiError(429, 'rate_limited', 'Write rate exceeded', {'Retry-After': '1'}) + self.tokens -= 1 + self.active += 1 + admitted = True + # Binary uploads are streamed by the authorized route into a bounded + # temporary file; do not copy a 10 MiB body into every API request. + if scope['method']=='POST' and scope['path'].endswith('/attachments'): + async with asyncio.timeout(self.cfg.request_timeout): + await self.app(scope,receive,tracked_send) + return + chunks, total = bytearray(), 0 + async with asyncio.timeout(self.cfg.request_timeout): + while True: + message = await receive() + if message['type'] == 'http.disconnect': + return + chunk = message.get('body', b'') + total += len(chunk) + if total > self.cfg.max_body_bytes: + raise ApiError(413, 'payload_too_large', 'Request body exceeds byte limit') + chunks.extend(chunk) + if not message.get('more_body', False): + break + body = bytes(chunks) + delivered = False + async def replay(): + nonlocal delivered + if delivered: + return await receive() + delivered = True + return {'type': 'http.request', 'body': body, 'more_body': False} + await self.app(scope, replay, tracked_send) + except ClientDisconnect: + status=499 + except TimeoutError: + if not started_response: + await error_response(ApiError(503, 'unavailable', 'Request timed out'), request_id)(scope, receive, tracked_send) + except ApiError as exc: + if not started_response: + await error_response(exc, request_id)(scope, receive, tracked_send) + except Exception as exc: + logger.error(json.dumps({'event': 'unexpected_error', 'request_id': request_id, 'type': type(exc).__name__})) + if not started_response: + await error_response(ApiError(500, 'internal_error', 'Internal error'), request_id)(scope, receive, tracked_send) + finally: + if admitted: + self.active -= 1 + # No query, body, headers, usernames, or message content in logs. + route = scope.get('route') + logger.info(json.dumps({'event': 'http', 'request_id': request_id, 'method': scope['method'], + 'route': getattr(route, 'path', 'unmatched'), 'status': status, + 'duration_ms': round((time.monotonic() - started) * 1000, 3)})) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..a66ac916 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,28 @@ +"""Versioned, transactional migrations, serialized across deployers.""" +import hashlib +import os +from pathlib import Path +import psycopg + + +def main(): + with psycopg.connect(os.environ['DATABASE_URL'],autocommit=True,connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_lock(23012026)') + try: + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())') + for file in sorted((Path(__file__).resolve().parents[1]/'migrations').glob('*.sql')): + data=file.read_bytes(); digest=hashlib.sha256(data).hexdigest() + row=conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s',(file.name,)).fetchone() + if row: + if row[0]!=digest: raise RuntimeError('An applied migration was modified: '+file.name) + print('Already applied:',file.name,flush=True) + continue + with conn.transaction(): + conn.execute(data.decode('utf-8')) + conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)',(file.name,digest)) + print('Applied:',file.name,flush=True) + finally: + conn.execute('SELECT pg_advisory_unlock(23012026)') + + +if __name__=='__main__': main() diff --git a/src/offload.py b/src/offload.py new file mode 100644 index 00000000..24842d33 --- /dev/null +++ b/src/offload.py @@ -0,0 +1,16 @@ +"""Keep resource leases until a blocking operation actually finishes on cancellation.""" +import asyncio + + +async def complete_thread(function, *args): + task = asyncio.create_task(asyncio.to_thread(function, *args)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + # asyncio cancellation cannot stop a Python thread. Do not close its file + # or release its decoder/upload permit while it is still using them. + try: + await asyncio.shield(task) + except Exception: + pass + raise diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..71b53cb7 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,101 @@ +"""Dry-run by default. Repairs visibility, never invents lost bytes or deletes referenced files.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +import time + +from botocore.exceptions import ClientError +import psycopg +from psycopg.rows import dict_row +from .storage import Storage, HEALTH_KEY + + +def _run(storage, apply, grace_seconds): + if grace_seconds < 60: + raise ValueError('Safety grace must be at least 60s (request deadline is 30s)') + started=time.perf_counter() + result={'apply':apply,'grace_seconds':grace_seconds,'objects_scanned':0,'metadata_scanned':0, + 'fresh_skipped':0,'busy_skipped':0,'actions':[]} + cutoff=datetime.now(timezone.utc).timestamp()-grace_seconds + with psycopg.connect(os.environ['DATABASE_URL'],row_factory=dict_row,autocommit=True, + options='-c statement_timeout=10000 -c lock_timeout=1000') as db: + for page in storage.client.get_paginator('list_objects_v2').paginate(Bucket=storage.bucket,Prefix='originals/'): + for obj in page.get('Contents',[]): + result['objects_scanned']+=1 + key=obj['Key'] + if key == HEALTH_KEY: continue # Reserved persistent readiness probe. + if obj['LastModified'].timestamp()>=cutoff: + result['fresh_skipped']+=1 + continue + with db.transaction(): + lock=db.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS acquired',(key,)).fetchone() + if not lock['acquired']: + result['busy_skipped']+=1 + continue + if db.execute('SELECT 1 FROM attachments WHERE object_key=%s',(key,)).fetchone(): + continue + # Recheck actual modification time under lock (stale listing is insufficient). + try: + head=storage.client.head_object(Bucket=storage.bucket,Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404','NoSuchKey'): continue + raise + if head['LastModified'].timestamp()>=cutoff: continue + result['actions'].append({'action':'delete_orphan','key':key}) + if apply: storage.delete(key) + # Keyset scan keeps memory bounded even for a large metadata table. + last=None + while True: + rows=db.execute('SELECT * FROM attachments WHERE (%s::uuid IS NULL OR id>%s::uuid) ORDER BY id LIMIT 100',(last,last)).fetchall() + if not rows: break + for row in rows: + result['metadata_scanned']+=1 + key=row['object_key'] + with db.transaction(): + lock=db.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS acquired',(key,)).fetchone() + if not lock['acquired']: + result['busy_skipped']+=1 + continue + try: + head=storage.client.head_object(Bucket=storage.bucket,Key=key) + present=head['ContentLength']==row['size_bytes'] and head.get('Metadata',{}).get('sha256')==row['sha256'] + except ClientError as exc: + if exc.response['Error']['Code'] not in ('404','NoSuchKey'): raise + present=False + if not present and row['storage_state']=='ready': + result['actions'].append({'action':'quarantine_missing','id':str(row['id'])}) + if apply: db.execute("UPDATE attachments SET storage_state='missing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + elif present and row['storage_state']=='missing': + # A repaired object must match actual bytes, not only user metadata. + obj=storage.get(key) + digest=hashlib.sha256() + size=0 + try: + while block:=obj['Body'].read(65536): + digest.update(block); size+=len(block) + if size>row['size_bytes']: break + finally: obj['Body'].close() + if size==row['size_bytes'] and digest.hexdigest()==row['sha256']: + result['actions'].append({'action':'restore_ready','id':str(row['id'])}) + if apply: db.execute("UPDATE attachments SET storage_state='ready',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + last=rows[-1]['id'] + result['elapsed_seconds']=round(time.perf_counter()-started,6) + return result + + +def run(apply=False, grace_seconds=600): + storage=Storage() + try: + return _run(storage,apply,grace_seconds) + finally: + storage.client.close() + + +if __name__=='__main__': + p=argparse.ArgumentParser() + p.add_argument('--apply',action='store_true') + p.add_argument('--grace-seconds',type=int,default=600) + args=p.parse_args() + print(json.dumps(run(args.apply,args.grace_seconds),indent=2)) diff --git a/src/reconcile_loop.py b/src/reconcile_loop.py new file mode 100644 index 00000000..706bb427 --- /dev/null +++ b/src/reconcile_loop.py @@ -0,0 +1,18 @@ +"""Periodic repair with a 10-minute grace; manual dry-run remains available.""" +import json +import signal +import threading +from .reconcile import run + +stop=threading.Event() +signal.signal(signal.SIGTERM,lambda *_:stop.set()) +signal.signal(signal.SIGINT,lambda *_:stop.set()) +while not stop.is_set(): + try: + result=run(apply=True,grace_seconds=600) + print(json.dumps({'event':'reconciliation','actions':len(result['actions']), + 'objects':result['objects_scanned'],'metadata':result['metadata_scanned'], + 'elapsed_seconds':result['elapsed_seconds']}),flush=True) + except Exception as exc: + print(json.dumps({'event':'reconciliation_failed','type':type(exc).__name__}),flush=True) + stop.wait(60) diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..71d90730 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,74 @@ +"""Bounded S3 calls and strict PNG/JPEG validation; never trust a client filename.""" +import hashlib +import os +import warnings + +import boto3 +from botocore.config import Config +from PIL import Image, UnidentifiedImageError + +from .common import ApiError + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +HEALTH_KEY = 'originals/.storage-health' +HEALTH_BYTES = b'messenger-storage-ready-v1' +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +warnings.simplefilter('error', Image.DecompressionBombWarning) + + +def validate_image(file, content_type): + try: + file.seek(0) + with Image.open(file, formats=['PNG', 'JPEG']) as image: + actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}[image.format] + if actual != content_type: + raise ApiError(415, 'unsupported_media_type', 'Image content does not match MIME') + if image.width * image.height > MAX_PIXELS: + raise ApiError(413, 'payload_too_large', 'Too many decoded pixels') + image.verify() + file.seek(0) + with Image.open(file, formats=['PNG', 'JPEG']) as image: + image.load() # JPEG.verify alone does not detect a truncated entropy stream. + file.seek(0) + digest = hashlib.file_digest(file, 'sha256').hexdigest() + file.seek(0) + return digest + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise ApiError(413, 'payload_too_large', 'Too many decoded pixels') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise ApiError(415, 'unsupported_media_type', 'Invalid PNG/JPEG') from None + + +class Storage: + def __init__(self): + self.bucket = os.environ['S3_BUCKET'] + self.client = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'], + aws_access_key_id=os.environ['S3_ACCESS_KEY'], + aws_secret_access_key=os.environ['S3_SECRET_KEY'], + region_name='us-east-1', + config=Config(connect_timeout=2, read_timeout=4, + retries={'total_max_attempts': 1}, + max_pool_connections=8, + s3={'addressing_style': 'path'})) + + def ready(self): + # Listing proves only filer metadata readiness. After a SeaweedFS restart, + # volume registration can lag; require a real read from a persisted volume. + obj = self.get(HEALTH_KEY) + try: + if obj['Body'].read(len(HEALTH_BYTES)+1) != HEALTH_BYTES: + raise ApiError(503, 'unavailable', 'Storage probe unavailable') + finally: + obj['Body'].close() + + def put(self, key, file, size, content_type, digest): + file.seek(0) + self.client.put_object(Bucket=self.bucket, Key=key, Body=file, ContentLength=size, + ContentType=content_type, Metadata={'sha256': digest}) + + def get(self, key): + return self.client.get_object(Bucket=self.bucket, Key=key) + + def delete(self, key): + self.client.delete_object(Bucket=self.bucket, Key=key) diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..4b07fdb0 --- /dev/null +++ b/src/store.py @@ -0,0 +1,169 @@ +import hashlib +import asyncio +import json +import logging +from uuid import uuid4 + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from .common import ApiError, encode_cursor, invalid + + +def clean(row): + if row is None: return None + result={} + for k,v in row.items(): + if hasattr(v,'isoformat'): v=v.isoformat(timespec='microseconds').replace('+00:00','Z') + elif isinstance(v,list): v=[str(x) for x in v] + elif type(v).__name__=='UUID': v=str(v) + result[k]=v + return result + + +class Store: + def __init__(self,cfg): + self.cfg=cfg + self.secret=cfg.cursor_secret.encode() + self.pool=AsyncConnectionPool(cfg.database_url,min_size=1,max_size=cfg.db_pool_size, + timeout=2,max_waiting=32,open=False, + kwargs={'row_factory':dict_row,'connect_timeout':3, + 'options':'-c statement_timeout=2000 -c lock_timeout=1000 -c timezone=UTC'}) + self.cache=Redis.from_url(cfg.redis_url,socket_timeout=.2,socket_connect_timeout=.2, + max_connections=8,decode_responses=True) + + async def open(self): await self.pool.open(wait=True,timeout=20) + async def close(self): + await self.cache.aclose() + await self.pool.close() + + async def ready(self): + async with self.pool.connection() as c: await c.execute('SELECT 1') + + async def register(self,username,display_name,password_hash): + try: + async with self.pool.connection() as c: + row=await (await c.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at', + (uuid4(),username,display_name,password_hash))).fetchone() + return clean(row) + except psycopg.errors.UniqueViolation: + raise ApiError(409,'conflict','Username already exists') from None + + async def credentials(self,username): + async with self.pool.connection() as c: + return await (await c.execute('SELECT * FROM users WHERE username=%s',(username,))).fetchone() + + async def create_session(self,user_id,token): + async with self.pool.connection() as c: + row=await (await c.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp()+%s*interval '1 second') RETURNING expires_at", + (hashlib.sha256(token.encode()).hexdigest(),user_id,self.cfg.session_ttl_seconds))).fetchone() + return clean(row)['expires_at'] + + async def authenticate(self,token): + async with self.pool.connection() as c: + session=await (await c.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp()', + (hashlib.sha256(token.encode()).hexdigest(),))).fetchone() + if session is None: raise ApiError(401,'unauthorized','Invalid session',{'WWW-Authenticate':'Bearer realm="messenger"'}) + key='user:'+str(session['user_id']) + try: + cached=await self.cache.get(key) + if cached: return json.loads(cached) + except (RedisError,ValueError): + logging.getLogger('messenger').warning('{"event":"cache_bypass"}') + row=await (await c.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s',(session['user_id'],))).fetchone() + user=clean(row) + try: await self.cache.set(key,json.dumps(user),ex=60) + except RedisError: pass + return user + + async def logout(self,token): + async with self.pool.connection() as c: + await c.execute('DELETE FROM sessions WHERE token_hash=%s',(hashlib.sha256(token.encode()).hexdigest(),)) + + async def _chat(self,c,actor,chat_id): + row=await (await c.execute('SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY user_id) AS member_ids FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s',(chat_id,actor))).fetchone() + if row is None: raise ApiError(404,'not_found','Object not found') + return clean(row) + + async def chat(self,actor,chat_id): + async with self.pool.connection() as c: return await self._chat(c,actor,chat_id) + + async def create_chat(self,actor,title,members): + if actor not in members: invalid('Members must include the current user') + try: + async with self.pool.connection() as c: + cid=uuid4() + await c.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',(cid,title)) + async with c.cursor() as cur: + await cur.executemany('INSERT INTO members(chat_id,user_id) VALUES (%s,%s)',[(cid,m) for m in members]) + return await self._chat(c,actor,cid) + except psycopg.errors.ForeignKeyViolation: + invalid('All members must exist') + + async def create_message(self,actor,chat_id,text,key,attachment_ids): + digest=hashlib.sha256(json.dumps({'text':text,'attachment_ids':attachment_ids},sort_keys=True,ensure_ascii=False,separators=(',',':')).encode()).hexdigest() + async with self.pool.connection() as c: + await self._chat(c,actor,chat_id) + # Fixed operation POST /chats/{id}/messages; lock and both rows share one transaction. + scope='POST:'+chat_id+':'+actor+':'+key + await c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(scope,)) + previous=await (await c.execute("SELECT request_hash,message_id FROM idempotency WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()",(actor,chat_id,key))).fetchone() + if previous: + if previous['request_hash']!=digest: raise ApiError(409,'conflict','Key was used for another message') + return clean(await (await c.execute('SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE id=%s',(previous['message_id'],))).fetchone()) + for aid in attachment_ids: + attachment=await (await c.execute('SELECT storage_state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE',(aid,chat_id))).fetchone() + if attachment is None: raise ApiError(404,'not_found','Object not found') + if attachment['storage_state']!='ready': raise ApiError(503,'unavailable','Object temporarily unavailable') + await c.execute('DELETE FROM idempotency WHERE actor_id=%s AND chat_id=%s AND key=%s',(actor,chat_id,key)) + row=await (await c.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING id,chat_id,sender_id,text,created_at,attachment_ids', + (uuid4(),chat_id,actor,text,attachment_ids))).fetchone() + for aid in attachment_ids: + await c.execute('INSERT INTO message_attachments(message_id,attachment_id) VALUES (%s,%s)',(row['id'],aid)) + await c.execute("INSERT INTO idempotency(actor_id,chat_id,key,request_hash,message_id,expires_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",(actor,chat_id,key,digest,row['id'])) + return clean(row) + + async def page(self,actor,chat_id,limit,cursor): + async with self.pool.connection() as c: + if chat_id: + await self._chat(c,actor,chat_id) + query='SELECT id,chat_id,sender_id,text,created_at,attachment_ids FROM messages WHERE chat_id=%s' + args=[chat_id]; resource='messages:'+chat_id + if cursor: + query+=' AND (created_at,id)>(%s::timestamptz,%s::uuid)'; args.extend(cursor) + query+=' ORDER BY created_at,id LIMIT %s'; args.append(limit+1) + else: + query='SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY user_id) AS member_ids FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s' + args=[actor]; resource='chats:'+actor + if cursor: + query+=' AND (c.created_at,c.id)>(%s::timestamptz,%s::uuid)'; args.extend(cursor) + query+=' ORDER BY c.created_at,c.id LIMIT %s'; args.append(limit+1) + rows=[clean(row) for row in await (await c.execute(query,args)).fetchall()] + selected=rows[:limit] + next_cursor=encode_cursor(resource,(selected[-1]['created_at'],selected[-1]['id']),self.secret) if len(rows)>limit else None + return {'items':selected,'next_cursor':next_cursor} + + async def attachment(self,actor,identifier): + async with self.pool.connection() as c: + row=await (await c.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(identifier,actor))).fetchone() + if row is None: raise ApiError(404,'not_found','Object not found') + if row['storage_state']!='ready': raise ApiError(503,'unavailable','Object temporarily unavailable') + return clean(row) + + async def save_attachment(self,actor,chat_id,filename,mime,size,digest,file,storage): + identifier=uuid4() + key='originals/'+str(identifier) + # The same lock is taken by reconciliation before it can delete an orphan. + # DB COMMIT uncertainty must NEVER trigger an unconditional S3 delete: the row + # may have committed. Leave a possible orphan for the grace-period reconciler. + async with self.pool.connection() as c: + await self._chat(c,actor,chat_id) + await c.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,)) + from .offload import complete_thread + await complete_thread(storage.put,key,file,size,mime,digest) + row=await (await c.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', + (identifier,chat_id,actor,filename,key,mime,size,digest))).fetchone() + return clean(row)