ЛР 5 — Группа БВТ2304 — Аккуратов — на 5 #17
+8
-19
@@ -1,25 +1,14 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
API_CPU_LIMIT=1.0
|
||||
API_MEMORY_LIMIT=512M
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
MAX_CONCURRENT_UPLOADS=10
|
||||
WORKER_CONCURRENCY=1
|
||||
WORKER_REPLICAS=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
RECONCILE_INTERVAL_SECONDS=900
|
||||
RECONCILE_GRACE_SECONDS=3600
|
||||
|
||||
@@ -4,15 +4,31 @@
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: Перенесены собственные сервер, PostgreSQL, S3, Compose-конфигурация и базовые тесты из предыдущей лабораторной. Сохранены переходы API по контракту lab5: загрузка вложения возвращает 202 и обрабатывается асинхронно. Добавлены RabbitMQ publisher/worker, transactional outbox, retry/recovery, versioned pipeline v1/v2, fencing устаревших задач, безопасный replay failed-вложений с audit trail и fault-injection тесты. README, COURSE, contracts и публичные tests задания не заменялись.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone:
|
||||
```sh
|
||||
git clone https://gitea.privcloud.tech/kaktuz2000/mtusi2026lab5.git
|
||||
cd mtusi2026lab5
|
||||
git switch solution/bvt2304-akkuratov
|
||||
cp .env.example .env
|
||||
scripts/bootstrap-secrets.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
until curl --fail --silent http://localhost:${APP_PORT:-8080}/health/ready >/dev/null; do sleep 2; done
|
||||
make check
|
||||
make test BASE_URL=http://localhost:${APP_PORT:-8080}
|
||||
```
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md: [REPORT.md](../../REPORT.md). `make check` — PASS; публичный smoke `make test BASE_URL=http://localhost:18080` — PASS (`PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.`). Также PASS: 8 сценариев fault injection, outbox/worker recovery, replay v1→v2 и failed replay с двумя audit-событиями, повреждённые вложения и S3 ACL, persistence, API failover. Фактические результаты: [evidence/lab5-integration.json](../../evidence/lab5-integration.json), [evidence/pipeline-faults.json](../../evidence/pipeline-faults.json), [evidence/load.json](../../evidence/load.json).
|
||||
|
||||
Ограничения / известные ошибки: Замер нагрузки — 30-секундный closed-loop тест на одной машине, без отдельного прогрева; он не является оценкой open-loop или production capacity. Другие известные ошибки по выполненным проверкам не обнаружены.
|
||||
|
||||
Источники и использование ИИ: Перенесён собственный код из mtusi2026lab4; требования и публичные проверки взяты из README.md, COURSE.md, CONTRIBUTING.md и tests репозитория mtusi2026lab5. Использован ИИ-ассистент для переноса, адаптации реализации, написания дополнительных тестов и отчёта. Внешние фрагменты кода не добавлялись.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY . /app
|
||||
RUN chown -R app:app /app
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["python3", "src/server.py"]
|
||||
@@ -4,10 +4,11 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test test-attachments test-idempotency-race test-reconcile test-worker test-pipeline-faults test-pipeline-replay test-outbox-recovery test-worker-recovery \
|
||||
test-reconcile-integration test-upload-concurrency load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "check/validate: offline contract checks; test: public smoke; student targets: focused attachment/pipeline checks"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -18,6 +19,36 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments:
|
||||
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-idempotency-race:
|
||||
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile:
|
||||
$(PYTHON) student-tests/test_reconcile.py
|
||||
|
||||
test-worker:
|
||||
$(PYTHON) student-tests/test_worker.py
|
||||
|
||||
test-pipeline-faults:
|
||||
$(PYTHON) student-tests/test_pipeline_faults.py
|
||||
|
||||
test-pipeline-replay:
|
||||
$(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-outbox-recovery:
|
||||
$(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-worker-recovery:
|
||||
$(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile-integration:
|
||||
$(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-upload-concurrency:
|
||||
$(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,37 +1,86 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Группа и фамилия: BVT2304, Akkuratov.
|
||||
- Целевая оценка: 5.
|
||||
- Выбранный трек: 2 — доказанная идемпотентность pipeline.
|
||||
- Commit/PR: не создан.
|
||||
- Использованные библиотеки: psycopg, redis, boto3, Pillow, pika; версии закреплены в `requirements.txt`.
|
||||
- В работе использовался ИИ-ассистент для переноса, адаптации и проверки кода.
|
||||
- Проверенный стенд: Python 3.12.7; PostgreSQL 16.4; RabbitMQ 3.13.7; SeaweedFS 4.51; HAProxy 3.0.7; Docker Desktop 28.0.4; 2 API-реплики. Хост: Apple M2 Pro (10 ядер, 16 GiB RAM), macOS 27.0.1.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Python 3.9+ и Docker Compose v2. Полные команды есть в [code.md](./code.md). Основной запуск:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
scripts/bootstrap-secrets.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
until curl --fail --silent http://localhost:8080/health/ready >/dev/null; do sleep 2; done
|
||||
make test BASE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
Секреты создаются локально и не публикуются. Миграции выполняются сервисом `migrate`; остановка без удаления данных — `docker compose down`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
PostgreSQL — источник истины для предметных данных, состояния вложения и transactional outbox. Подтверждённая API-загрузка сохраняет исходник в S3 staging и метаданные с outbox-записью до ответа `202 queued`. Publisher передаёт durable RabbitMQ job с ID и служебными метаданными, без bytes и credentials. Worker декодирует изображение в отдельном процессе с ограничением времени, записывает оригинал и `thumbnail`/`crop`/`optimized`, затем фиксирует `ready`. Повторы ограничены, используют backoff с jitter; lease и publisher восстанавливают просроченную работу.
|
||||
|
||||
## Доказательства критериев
|
||||
Обработка ограничена `JOB_TIMEOUT_SECONDS`; дочерний процесс принудительно завершается при превышении. S3-клиент задаёт connect/read timeout и число retries, lease включает отдельный сетевой бюджет. Worker имеет memory limit 1 GiB и concurrency по умолчанию 1; API ограничивает файл 10 MiB и изображение 40 млн пикселей.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Версии pipeline задают отдельные параметры и S3-prefix для вариантов: v1 использует JPEG quality 85/WebP 80, v2 — JPEG 80/WebP 75. Геометрия остаётся одинаковой. Операторский replay создаёт новый `job_id`, сбрасывает попытку, публикует новую outbox-запись и добавляет audit event. SQL completion/failure ограничены текущими job ID, pipeline version и attempt; поздняя задача не может изменить текущее состояние. Reconciler периодически сверяет PostgreSQL и S3. Filer доступен только из внутренней Compose-сети.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Трек 2: гипотеза и методика
|
||||
|
||||
## Эксперименты и отказы
|
||||
**Гипотеза:** повторная доставка может повторить физические записи, но уникальные versioned object keys и fencing по `(attachment_id, job_id, pipeline_version, attempt)` не допускают потери принятой работы или публикации устаревшего результата.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Методика: `make test-pipeline-faults` запускает fault injection на штатном consumer/publisher-коде и контролируемых брокере, объектном хранилище и state-store. Используется публичный синтетический fixture `sample.png`. Падения вводятся до первой побочной записи, после записи оригинала и после фиксации результата до ack; дополнительно проверяются повторы, reorder, смена версии и неопределённый publisher confirm. Результат каждого запуска записывается в [evidence/pipeline-faults.json](./evidence/pipeline-faults.json).
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Сценарий | Наблюдение |
|
||||
| --- | --- |
|
||||
| Дубль после commit до ack | 2 доставки, 4 записи объектов для одного набора; повторная доставка — 0 дополнительных записей |
|
||||
| Crash сразу после claim | lease recovery создал attempt 2; повтор завершился `ready` |
|
||||
| Crash сразу после записи оригинала | повторная запись детерминированна; attempt 2 завершился `ready` |
|
||||
| Reorder attempt 2 затем attempt 1 | устаревшая попытка: 0 дополнительных записей |
|
||||
| v1 superseded версией v2 во время обработки | metadata ссылается только на `v2` variants |
|
||||
| Publisher crash до publish | pending outbox пережил перезапуск, задание опубликовано после восстановления |
|
||||
| Confirm потерян после broker accept | два сообщения имеют идентичное тело и стабильный message ID; worker no-op для дубля |
|
||||
| Ограничение времени transform | зависший дочерний процесс завершён за 1.008 секунды при бюджете 1 секунда |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Compose integration PASS: `make test-outbox-recovery` подтвердил переход queued→ready после принятого upload при остановленном publisher; `make test-pipeline-replay` выполнил v1→v2, повторно обработал failed-вложение на той же версии и проверил два audit-события. Исходный checksum сохранился. `make test-worker-recovery` подтвердил queued-вложение после перезапуска API/worker. Фактические результаты проверок стенда сохранены в [evidence/lab5-integration.json](./evidence/lab5-integration.json).
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Нагрузка
|
||||
|
||||
Повторяемый замер: `.venv/bin/python tests/load.py --base-url http://localhost:18080 --duration 30 --concurrency 4 --output evidence/load.json`. После готовности API и подготовки тестовых клиентов выполнен 30-секундный closed-loop POST-тест на синтетических сообщениях: 18,768 успешных запросов, 0 ошибок, 625.458 успешных RPS; latency успешных запросов — p50 5.816 ms, p95 8.820 ms, p99 19.155 ms. Дополнительно `docker stats` измерял CPU и память контейнеров Compose: сумма CPU peak/mean — 322.07%/232.49%, сумма памяти peak/mean — 890,263,039/814,641,644 байт, 11 выборок (примерно раз в 2.73 секунды). API: две реплики, лимит каждой 1 CPU/512 MiB; worker memory limit 1 GiB. Отдельная предварительная фаза прогрева не проводилась. Это closed-loop результат на указанном хосте; его нельзя напрямую трактовать как open-loop capacity или гарантированный production throughput. Полные данные: [evidence/load.json](./evidence/load.json).
|
||||
|
||||
## Инварианты и доставка
|
||||
|
||||
- `202` означает, что исходник и намерение обработки устойчиво сохранены; publisher можно запускать позже.
|
||||
- Job повторяется at-least-once. После потери confirm одно сообщение может быть опубликовано дважды; тест намеренно наблюдает две доставки.
|
||||
- Результаты представлены детерминированными S3 keys, различными для pipeline versions; metadata переключается только условным SQL update.
|
||||
- Ack отправляется только после устойчивого результата или после записи долговременного retry/outbox намерения. Дубль уже завершённой задачи — no-op.
|
||||
- Это не exactly-once delivery: между broker accept и фиксацией publisher confirm возможна повторная публикация, а между SQL commit результата и consumer ack возможна повторная доставка. Гарантируется fencing/idempotency эффекта, не единственная доставка или единственное выполнение.
|
||||
|
||||
## Проверки
|
||||
|
||||
| Проверка | Команда | Результат |
|
||||
| --- | --- | --- |
|
||||
| Контракт skeleton | `make check` | PASS |
|
||||
| OpenAPI/JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS |
|
||||
| Варианты v1/v2 | `make test-worker PYTHON=.venv/bin/python` | PASS |
|
||||
| Reconciler unit tests | `make test-reconcile PYTHON=.venv/bin/python` | PASS |
|
||||
| Track 2 injection | `make test-pipeline-faults PYTHON=.venv/bin/python` | PASS; 8 сценариев |
|
||||
| Python syntax/whitespace | `compileall`, `git diff --check` | PASS |
|
||||
| Compose parse | `docker compose --env-file .env.example config --quiet` | PASS |
|
||||
| Public HTTP smoke | `make test BASE_URL=http://localhost:18080` | PASS |
|
||||
| Ошибочные загрузки/ACL | `make test-attachments BASE_URL=http://localhost:18080` | PASS |
|
||||
| Concurrent idempotency | `make test-idempotency-race BASE_URL=http://localhost:18080` | PASS; 12 запросов, 2 API, одно сообщение |
|
||||
| Outbox после publisher stop | `make test-outbox-recovery BASE_URL=http://localhost:18080` | PASS; queued→ready без ручной публикации |
|
||||
| Очередь после API/worker restart | `make test-worker-recovery BASE_URL=http://localhost:18080` | PASS; ссылка сообщения и оригинал сохранены |
|
||||
| Реальный replay v1→v2 | `make test-pipeline-replay BASE_URL=http://localhost:18080` | PASS; разные thumbnail SHA-256, оригинал совпал |
|
||||
| Повтор failed-вложения и audit | `make test-pipeline-replay BASE_URL=http://localhost:18080` | PASS; failed→queued→ready, два audit-события |
|
||||
| Reconciliation/S3 ACL | `make test-reconcile-integration BASE_URL=http://localhost:18080` | PASS; orphan cleanup, exact restore, анонимный GET запрещён |
|
||||
| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080` | PASS; данные и сессия пережили restart |
|
||||
| API failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080` | PASS; 131 запись, восстановление за 0.681 s |
|
||||
| Параллельная загрузка | `make test-upload-concurrency PYTHON=.venv/bin/python BASE_URL=http://localhost:18080` | PASS; 10 × 5,883,411 байт |
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Запуск и проверка лабораторной 5
|
||||
|
||||
## Подготовка стенда
|
||||
|
||||
Нужны Docker Compose v2 и доступный Docker daemon. Все команды выполняются из корня репозитория.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
scripts/bootstrap-secrets.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
```
|
||||
|
||||
Секреты создаются локально в `secrets/` с правами владельца и не выводятся в терминал. Не добавляйте `.env` или `secrets/` в Git. Compose поднимает PostgreSQL, RabbitMQ, SeaweedFS, миграции, API, publisher, worker, периодический reconciler и HAProxy.
|
||||
|
||||
Дождитесь готовности API и проверьте состояние сервисов:
|
||||
|
||||
```sh
|
||||
until curl --fail --silent http://localhost:8080/health/ready >/dev/null; do sleep 2; done
|
||||
docker compose ps
|
||||
curl --fail --silent http://localhost:8080/health/ready
|
||||
```
|
||||
|
||||
Если порт `8080` занят, поменяйте `APP_PORT` в локальном `.env` (например, на `18080`) и используйте этот порт во всех командах ниже.
|
||||
|
||||
## Проверки кода
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
make check PYTHON=.venv/bin/python
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make test-worker PYTHON=.venv/bin/python
|
||||
make test-reconcile PYTHON=.venv/bin/python
|
||||
make test-pipeline-faults PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
`test-pipeline-faults` исполняет fault injection для дубля, reorder, падения после claim и после записи оригинала, потери ack, гонки старой/новой версии, остановки publisher до publish, потери publisher confirm и hard timeout обработки. Санитизированный JSON создаётся в `evidence/pipeline-faults.json`.
|
||||
|
||||
## API и интеграционные проверки
|
||||
|
||||
Стенд должен быть запущен. Тесты создают синтетические данные и не удаляют их; используйте учебный стенд.
|
||||
|
||||
```sh
|
||||
make test BASE_URL=http://localhost:8080
|
||||
make test-attachments BASE_URL=http://localhost:8080
|
||||
make test-idempotency-race BASE_URL=http://localhost:8080
|
||||
make test-reconcile-integration BASE_URL=http://localhost:8080
|
||||
make test-upload-concurrency BASE_URL=http://localhost:8080
|
||||
make load BASE_URL=http://localhost:8080
|
||||
# Exact 30-second/4-client benchmark, saving sanitized metrics:
|
||||
.venv/bin/python tests/load.py --base-url http://localhost:8080 \
|
||||
--duration 30 --concurrency 4 --output evidence/load.json
|
||||
```
|
||||
|
||||
Для проверок уровня 4/5:
|
||||
|
||||
```sh
|
||||
# Подтверждённая API загрузка при остановленном publisher должна дойти до ready после его старта.
|
||||
make test-outbox-recovery BASE_URL=http://localhost:8080
|
||||
|
||||
# Upload/message при остановленном worker; restart API и worker, затем ready и download.
|
||||
make test-worker-recovery BASE_URL=http://localhost:8080
|
||||
|
||||
# Готовое вложение переобрабатывается pipeline v2 с новой версией job.
|
||||
make test-pipeline-replay BASE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
Pipeline v1 использует JPEG quality 85 и WebP quality 80; v2 — JPEG quality 80 и WebP quality 75. Геометрия вариантов остаётся фиксированной контрактом. При replay `job_id` меняется, попытки начинаются заново, версии вариантов получают разные S3-prefix (`variants/<attachment-id>/vN/`), а событие записывается в PostgreSQL. В рабочем контейнере явный replay выполняется так:
|
||||
|
||||
```sh
|
||||
docker compose exec --index 1 -T api python3 -m src.replay_attachment \
|
||||
--attachment-id "$ATTACHMENT_ID" --pipeline-version 2
|
||||
```
|
||||
|
||||
Перед запуском задайте `ATTACHMENT_ID` равным UUID существующего вложения из ответа API.
|
||||
Для повторной попытки failed-вложения укажите его текущую версию; переход на уже опубликованную версию разрешён только для failed, повышение версии supersede-ит старую активную задачу.
|
||||
|
||||
## Отказ и сохранность
|
||||
|
||||
```sh
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
|
||||
docker compose up -d --scale api=2 api
|
||||
```
|
||||
|
||||
Сценарий failover оставляет выбранный API-контейнер остановленным; последняя команда возвращает две реплики. Restart hook перезапускает PostgreSQL, SeaweedFS, RabbitMQ, API, publisher и worker без удаления volumes.
|
||||
|
||||
## Диагностика и остановка
|
||||
|
||||
```sh
|
||||
docker compose logs -f api publisher worker rabbitmq reconciler
|
||||
docker compose down
|
||||
```
|
||||
|
||||
`docker compose down` сохраняет named volumes. Для намеренного удаления тестовой БД, очереди и S3-файлов:
|
||||
|
||||
```sh
|
||||
docker compose down --volumes
|
||||
```
|
||||
|
||||
Эту команду выполняйте только если данные стенда больше не нужны.
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.13.7-management-alpine
|
||||
secrets:
|
||||
- source: rabbitmq_config
|
||||
target: /etc/rabbitmq/conf.d/10-defaults.conf
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
s3-core:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
entrypoint: ["weed"]
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -filer=true
|
||||
- -s3=false
|
||||
volumes:
|
||||
- seaweed_data:/data
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- "curl --fail --silent http://127.0.0.1:8888/ >/dev/null"
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
networks:
|
||||
- seaweed_internal
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
entrypoint: ["weed"]
|
||||
command:
|
||||
- s3
|
||||
- -filer=s3-core:8888
|
||||
- -config=/run/secrets/s3_config
|
||||
- -port=8333
|
||||
- -port.https=0
|
||||
- -port.grpc=0
|
||||
- -port.iceberg=0
|
||||
- -port.lance=0
|
||||
- -concurrentFileUploadLimit=10
|
||||
- -concurrentUploadLimitMB=100
|
||||
- -readerCacheSizeMB=64
|
||||
- -autoCreateBucket=false
|
||||
- -ip.bind=0.0.0.0
|
||||
secrets:
|
||||
- s3_config
|
||||
depends_on:
|
||||
s3-core:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- default
|
||||
- seaweed_internal
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]"
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/bootstrap_s3.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger-attachments
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10}
|
||||
UPLOAD_TIMEOUT_SECONDS: "60"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_MAX_SIZE: "8"
|
||||
DB_POOL_TIMEOUT_SECONDS: "3"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
DB_LOCK_TIMEOUT_MS: "2000"
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python3
|
||||
- -c
|
||||
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPU_LIMIT:-1.0}"
|
||||
memory: "${API_MEMORY_LIMIT:-512M}"
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
publisher:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "-m", "src.publisher"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
secrets:
|
||||
- database_url
|
||||
- broker_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "-m", "src.worker"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
secrets:
|
||||
- database_url
|
||||
- broker_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
reconciler:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["sh", "scripts/reconcile-loop.sh"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900}
|
||||
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
image: haproxy:3.0.7-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: unless-stopped
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
s3_config:
|
||||
file: ./secrets/s3.json
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
rabbitmq_password:
|
||||
file: ./secrets/rabbitmq_password
|
||||
rabbitmq_config:
|
||||
file: ./secrets/rabbitmq.conf
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
seaweed_data:
|
||||
rabbitmq_data:
|
||||
|
||||
networks:
|
||||
seaweed_internal:
|
||||
internal: true
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"date": "2026-11-04",
|
||||
"compose": {
|
||||
"api_replicas": 2,
|
||||
"host_port": 18080,
|
||||
"services_healthy": true,
|
||||
"readiness": "200"
|
||||
},
|
||||
"fixture": {
|
||||
"file": "tests/fixtures/sample.png",
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"size_bytes": 111817
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"name": "public_smoke",
|
||||
"result": "PASS",
|
||||
"command": "make test BASE_URL=http://localhost:18080"
|
||||
},
|
||||
{
|
||||
"name": "attachment_negative_cases",
|
||||
"result": "PASS",
|
||||
"command": "make test-attachments BASE_URL=http://localhost:18080",
|
||||
"cases": [
|
||||
"damaged PNG",
|
||||
"spoofed MIME",
|
||||
"oversized upload",
|
||||
"pixel limit",
|
||||
"unsafe filename",
|
||||
"unauthorized chat",
|
||||
"cross-chat attachment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idempotency_race",
|
||||
"result": "PASS",
|
||||
"replicas": 2,
|
||||
"concurrent_same_key_requests": 12,
|
||||
"messages_created": 1
|
||||
},
|
||||
{
|
||||
"name": "outbox_recovery",
|
||||
"result": "PASS",
|
||||
"status_before_publisher_restart": "queued",
|
||||
"status_after_restart": "ready",
|
||||
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"
|
||||
},
|
||||
{
|
||||
"name": "worker_restart_recovery",
|
||||
"result": "PASS",
|
||||
"sequence": [
|
||||
"stop worker",
|
||||
"upload returns 202 queued",
|
||||
"link attachment to message",
|
||||
"restart API",
|
||||
"start worker",
|
||||
"attachment reaches ready"
|
||||
],
|
||||
"message_attachment_link_survived": true
|
||||
},
|
||||
{
|
||||
"name": "pipeline_version_replay",
|
||||
"result": "PASS",
|
||||
"version_transition": "1 -> 2",
|
||||
"job_id_changed": true,
|
||||
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"v1_thumbnail_sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
||||
"v2_thumbnail_sha256": "7904b78c14d936a9e5b4d024bd7bf9ffe4de546c484b356177056b8e949e763f"
|
||||
},
|
||||
{
|
||||
"name": "failed_pipeline_replay",
|
||||
"result": "PASS",
|
||||
"previous_job_id": "2628b822-ae4c-4784-b3bf-03f695176984",
|
||||
"failed_replay_job_id": "797cc24f-0c3a-49fa-9f25-febd21506c6c",
|
||||
"pipeline_version": 2,
|
||||
"audited_replays": 2,
|
||||
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"
|
||||
},
|
||||
{
|
||||
"name": "reconciliation",
|
||||
"result": "PASS",
|
||||
"dataset_objects_scanned": 36,
|
||||
"metadata_records_scanned": 9,
|
||||
"scan_duration_ms": 20.873,
|
||||
"orphan_detected_and_removed": 1,
|
||||
"missing_object_detected_and_restored": 1,
|
||||
"fresh_orphan_preserved_until_grace": true,
|
||||
"anonymous_s3_access": "denied"
|
||||
},
|
||||
{
|
||||
"name": "persistence_restart",
|
||||
"result": "PASS",
|
||||
"confirmed_data_preserved": true,
|
||||
"existing_session_preserved": true
|
||||
},
|
||||
{
|
||||
"name": "api_failover",
|
||||
"result": "PASS",
|
||||
"instances_before": 2,
|
||||
"successful_writes": 131,
|
||||
"failed_attempts": 1,
|
||||
"first_recovery_seconds": 0.681,
|
||||
"confirmed_data_preserved": true
|
||||
},
|
||||
{
|
||||
"name": "concurrent_upload_memory",
|
||||
"result": "PASS",
|
||||
"api_replicas": 2,
|
||||
"uploads": 10,
|
||||
"concurrency": 10,
|
||||
"bytes_per_upload": 5883411,
|
||||
"duration_seconds": 2.525,
|
||||
"baseline_max_container_bytes": 96101990,
|
||||
"peak_max_container_bytes": 96101990,
|
||||
"peak_delta_bytes": 0,
|
||||
"sampling_method": "continuous docker stats stream"
|
||||
}
|
||||
],
|
||||
"track_2_fault_injection": "pipeline-faults.json"
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"lab": 5,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.007,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 18768,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 625.458,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.816,
|
||||
"p95": 8.82,
|
||||
"p99": 19.155
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.",
|
||||
"environment": {
|
||||
"host": "Apple M2 Pro",
|
||||
"host_cpu_cores": 10,
|
||||
"host_ram_bytes": 17179869184,
|
||||
"macos_version": "27.0.1",
|
||||
"docker_desktop_version": "28.0.4",
|
||||
"api_replicas": 2,
|
||||
"api_cpu_limit_per_container": 1.0,
|
||||
"api_memory_limit_per_container": "512M",
|
||||
"worker_memory_limit": "1G"
|
||||
},
|
||||
"warmup": "No separate warm-up interval; run started after readiness and test-client bootstrap.",
|
||||
"dataset": "18,768 successful synthetic POST message writes; concurrency 4.",
|
||||
"resource_sampling": {
|
||||
"sample_interval_seconds_approx": 2.73,
|
||||
"samples": 11,
|
||||
"cpu_percent_sum_all_compose_containers": {
|
||||
"peak": 322.07,
|
||||
"mean": 232.49
|
||||
},
|
||||
"memory_bytes_sum_all_compose_containers": {
|
||||
"peak": 890263039,
|
||||
"mean": 814641644
|
||||
},
|
||||
"source": "docker stats --no-stream; aggregate across running Compose containers"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"fixture": "tests/fixtures/sample.png",
|
||||
"invariants": {
|
||||
"ambiguous_confirm_republishes_stable_job": true,
|
||||
"confirmed_work_eventually_ready": true,
|
||||
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
|
||||
"duplicate_effects_create_no_second_variant_set": true,
|
||||
"stale_attempt_writes_no_effects": true,
|
||||
"stale_pipeline_cannot_publish_current_metadata": true
|
||||
},
|
||||
"result": "PASS",
|
||||
"scenarios": [
|
||||
{
|
||||
"deliveries": 2,
|
||||
"duplicate_delivery_writes": 0,
|
||||
"effective_object_writes": 4,
|
||||
"scenario": "duplicate_delivery_after_commit_before_ack"
|
||||
},
|
||||
{
|
||||
"attempts": 2,
|
||||
"scenario": "crash_after_claim",
|
||||
"terminal_status": "ready"
|
||||
},
|
||||
{
|
||||
"attempts": 2,
|
||||
"scenario": "crash_after_original_put",
|
||||
"terminal_status": "ready"
|
||||
},
|
||||
{
|
||||
"current_attempt": 2,
|
||||
"scenario": "out_of_order_attempt_delivery",
|
||||
"stale_attempt_extra_writes": 0
|
||||
},
|
||||
{
|
||||
"current_pipeline_version": 2,
|
||||
"published_variant_versions": [
|
||||
"v2"
|
||||
],
|
||||
"scenario": "stale_pipeline_after_newer_version"
|
||||
},
|
||||
{
|
||||
"outbox_recovered_after_restart": true,
|
||||
"published_after_restart": 1,
|
||||
"scenario": "publisher_crash_before_publish"
|
||||
},
|
||||
{
|
||||
"deliveries": 2,
|
||||
"same_message_id": true,
|
||||
"scenario": "publish_confirm_lost_after_broker_accept"
|
||||
},
|
||||
{
|
||||
"configured_timeout_seconds": 1,
|
||||
"observed_seconds": 1.008,
|
||||
"scenario": "transform_timeout_terminated"
|
||||
}
|
||||
],
|
||||
"track": "2"
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option http-keep-alive
|
||||
timeout connect 2s
|
||||
timeout client 15s
|
||||
timeout server 15s
|
||||
timeout http-request 5s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
|
||||
frontend messenger
|
||||
bind :8080
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
option redispatch
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title varchar(100) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text varchar(4000) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method varchar(8) NOT NULL,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key varchar(128) NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
|
||||
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
|
||||
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
|
||||
@@ -0,0 +1,59 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank;
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
filename varchar(255) NOT NULL,
|
||||
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
staging_key text NOT NULL UNIQUE,
|
||||
job_id uuid NOT NULL UNIQUE,
|
||||
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
lease_until timestamptz,
|
||||
status varchar(16) NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'),
|
||||
error_code varchar(64),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, ordinal)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
CREATE TABLE attachment_outbox (
|
||||
id uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
attempt integer NOT NULL CHECK (attempt >= 1),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
UNIQUE (job_id, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_outbox_pending_idx
|
||||
ON attachment_outbox (available_at, created_at)
|
||||
WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE reconciliation_incidents (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')),
|
||||
first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (attachment_id, incident_type)
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
ALTER TABLE attachment_outbox
|
||||
DROP CONSTRAINT attachment_outbox_job_id_fkey;
|
||||
|
||||
CREATE TABLE attachment_pipeline_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
prior_job_id uuid NOT NULL,
|
||||
job_id uuid NOT NULL,
|
||||
prior_version integer NOT NULL CHECK (prior_version >= 1),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
action varchar(16) NOT NULL CHECK (action = 'replay'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_pipeline_events_attachment_created_idx
|
||||
ON attachment_pipeline_events (attachment_id, created_at);
|
||||
@@ -0,0 +1,6 @@
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-pool==3.3.3
|
||||
redis==5.2.1
|
||||
boto3==1.35.99
|
||||
Pillow==11.0.0
|
||||
pika==1.3.2
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRET_DIR = ROOT / "secrets"
|
||||
|
||||
|
||||
def main():
|
||||
SECRET_DIR.mkdir(mode=0o700, exist_ok=True)
|
||||
names = (
|
||||
"postgres_password",
|
||||
"database_url",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key",
|
||||
"s3_admin_access_key_id",
|
||||
"s3_admin_secret_access_key",
|
||||
"rabbitmq_password",
|
||||
"rabbitmq.conf",
|
||||
"broker_url",
|
||||
"s3.json",
|
||||
)
|
||||
existing = [name for name in names if (SECRET_DIR / name).exists()]
|
||||
if existing:
|
||||
raise SystemExit(
|
||||
"Secret files already exist; refusing to rotate credentials without a migration: "
|
||||
+ ", ".join(existing)
|
||||
)
|
||||
|
||||
db_password = secrets.token_hex(24)
|
||||
broker_password = secrets.token_urlsafe(32)
|
||||
values = {
|
||||
"postgres_password": db_password,
|
||||
"database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger",
|
||||
"s3_access_key_id": "mngr" + secrets.token_hex(8),
|
||||
"s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"s3_admin_access_key_id": "admin" + secrets.token_hex(7),
|
||||
"s3_admin_secret_access_key": secrets.token_urlsafe(40),
|
||||
"rabbitmq_password": broker_password,
|
||||
"broker_url": f"amqp://messenger:{broker_password}@rabbitmq:5672/%2f",
|
||||
}
|
||||
for name, value in values.items():
|
||||
path = SECRET_DIR / name
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(value + "\n")
|
||||
descriptor = os.open(
|
||||
SECRET_DIR / "rabbitmq.conf",
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o600,
|
||||
)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write("default_user = messenger\n")
|
||||
output.write(f"default_pass = {broker_password}\n")
|
||||
s3_config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "bootstrap-admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": values["s3_admin_access_key_id"],
|
||||
"secretKey": values["s3_admin_secret_access_key"],
|
||||
}
|
||||
],
|
||||
"actions": ["Admin"],
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": values["s3_access_key_id"],
|
||||
"secretKey": values["s3_secret_access_key"],
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"Write:messenger-attachments",
|
||||
"List:messenger-attachments",
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
json.dump(s3_config, output, separators=(",", ":"))
|
||||
output.write("\n")
|
||||
print("Local secrets generated with owner-only permissions; values were not printed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/bootstrap-secrets.py"
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
interval=${RECONCILE_INTERVAL_SECONDS:-900}
|
||||
grace=${RECONCILE_GRACE_SECONDS:-3600}
|
||||
|
||||
case "$interval" in
|
||||
''|*[!0-9]*)
|
||||
echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$grace" in
|
||||
''|*[!0-9]*)
|
||||
echo "RECONCILE_GRACE_SECONDS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
if [ "$interval" -lt 60 ]; then
|
||||
echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$grace" -lt 60 ]; then
|
||||
echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
while :
|
||||
do
|
||||
python3 src/reconcile.py --grace-seconds "$grace" --apply
|
||||
sleep "$interval"
|
||||
done
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres s3-core rabbitmq
|
||||
docker compose up -d --wait postgres s3-core rabbitmq
|
||||
docker compose restart s3
|
||||
docker compose up -d --wait s3
|
||||
attempt=0
|
||||
until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 4 ]; then
|
||||
echo "S3 object reads did not recover after restart" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
docker compose restart api
|
||||
docker compose restart publisher worker
|
||||
docker compose up -d --wait api publisher worker
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
found=
|
||||
for container_id in $(docker compose ps --quiet --status running api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$container_id
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
printf '%s\n' 'No running API container matched the requested instance alias.' >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop --time 5 "$found" >/dev/null
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API application package."""
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from .config import APP_HOST, APP_PORT
|
||||
from .handlers import MessengerHandler
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger")
|
||||
|
||||
|
||||
def run_server():
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
handler = partial(MessengerHandler, store=store)
|
||||
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
LOGGER.info("received signal %s, shutting down gracefully", signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
|
||||
run_server()
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3.bootstrap")
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
direct = os.getenv(name)
|
||||
if direct:
|
||||
return direct
|
||||
file_path = os.getenv(name + "_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError(f"Set {name} or {name}_FILE")
|
||||
|
||||
|
||||
def main():
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}),
|
||||
)
|
||||
bucket = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status != 404:
|
||||
raise
|
||||
client.create_bucket(Bucket=bucket)
|
||||
client.put_bucket_acl(Bucket=bucket, ACL="private")
|
||||
LOGGER.info("private S3 bucket bootstrap completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
|
||||
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
|
||||
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
|
||||
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
|
||||
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
|
||||
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
|
||||
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10"))
|
||||
UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60"))
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
|
||||
|
||||
def secret_value(name: str, default: str = "") -> str:
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv(name + "_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
if default:
|
||||
return default
|
||||
raise RuntimeError(f"Set {name} or {name}_FILE")
|
||||
|
||||
|
||||
S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY")
|
||||
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
if SESSION_TTL_SECONDS < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
|
||||
if IDEMPOTENCY_TTL_SECONDS < 86400:
|
||||
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
|
||||
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
|
||||
raise RuntimeError("Database timeouts must be positive")
|
||||
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
|
||||
raise RuntimeError("Database pool settings must be positive")
|
||||
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
|
||||
raise RuntimeError("HTTP limits must be positive")
|
||||
if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024:
|
||||
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB")
|
||||
if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000:
|
||||
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million")
|
||||
if MAX_CONCURRENT_UPLOADS < 1:
|
||||
raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive")
|
||||
if UPLOAD_TIMEOUT_SECONDS < 1:
|
||||
raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive")
|
||||
@@ -0,0 +1,26 @@
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class CacheUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyConflictError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AttachmentNotReadyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ObjectMissingError(Exception):
|
||||
pass
|
||||
+560
@@ -0,0 +1,560 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import tempfile
|
||||
import uuid
|
||||
import warnings
|
||||
import zlib
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
import unicodedata
|
||||
|
||||
import psycopg
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .config import (
|
||||
IDEMPOTENCY_KEY_RE,
|
||||
INSTANCE_ID,
|
||||
MAX_CONCURRENT_UPLOADS,
|
||||
MAX_IMAGE_PIXELS,
|
||||
MAX_INFLIGHT_REQUESTS,
|
||||
MAX_REQUEST_BODY_BYTES,
|
||||
MAX_UPLOAD_BYTES,
|
||||
UPLOAD_TIMEOUT_SECONDS,
|
||||
USER_RE,
|
||||
)
|
||||
from .errors import (
|
||||
ApiError,
|
||||
AttachmentNotReadyError,
|
||||
CacheUnavailableError,
|
||||
IdempotencyConflictError,
|
||||
ObjectMissingError,
|
||||
)
|
||||
from .security import hash_password, isoformat, valid_uuid
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger.http")
|
||||
|
||||
|
||||
class RequestGuard:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.inflight = 0
|
||||
|
||||
def acquire(self):
|
||||
with self.lock:
|
||||
if self.inflight >= MAX_INFLIGHT_REQUESTS:
|
||||
raise ApiError(503, "unavailable", "Server is busy")
|
||||
self.inflight += 1
|
||||
|
||||
def release(self):
|
||||
with self.lock:
|
||||
self.inflight = max(0, self.inflight - 1)
|
||||
|
||||
|
||||
REQUEST_GUARD = RequestGuard()
|
||||
UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
|
||||
IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
class MessengerHandler(BaseHTTPRequestHandler):
|
||||
def __init__(self, request, client_address, server, store: Store):
|
||||
self.store = store
|
||||
super().__init__(request, client_address, server)
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(15)
|
||||
|
||||
def log_message(self, format: str, *args):
|
||||
return
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method: str):
|
||||
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
acquired = False
|
||||
try:
|
||||
REQUEST_GUARD.acquire()
|
||||
acquired = True
|
||||
parsed = urlsplit(self.path)
|
||||
path = parsed.path
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if path == "/health/live" and method == "GET":
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/health/ready" and method == "GET":
|
||||
try:
|
||||
self.store.ready()
|
||||
except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError):
|
||||
raise ApiError(503, "unavailable", "A required storage service is unavailable")
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/api/v1/users" and method == "POST":
|
||||
self._handle_register(request_id)
|
||||
elif path == "/api/v1/auth/sessions" and method == "POST":
|
||||
self._handle_create_session(request_id)
|
||||
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||
self._handle_revoke_session(request_id)
|
||||
elif path == "/api/v1/users/me" and method == "GET":
|
||||
self._send_json(200, self._get_actor(request_id), request_id)
|
||||
elif path == "/api/v1/chats" and method == "POST":
|
||||
self._handle_create_chat(request_id)
|
||||
elif path == "/api/v1/chats" and method == "GET":
|
||||
self._handle_list_chats(query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST":
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
self._handle_upload_attachment(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/attachments/") and method == "GET":
|
||||
attachment_path = path.removeprefix("/api/v1/attachments/")
|
||||
if attachment_path.endswith("/content"):
|
||||
attachment_id = attachment_path.removesuffix("/content")
|
||||
self._handle_download_attachment(attachment_id, query, request_id)
|
||||
else:
|
||||
self._handle_get_attachment(attachment_path, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
if method == "POST":
|
||||
self._handle_create_message(chat_id, request_id)
|
||||
else:
|
||||
self._handle_list_messages(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and method == "GET":
|
||||
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
|
||||
else:
|
||||
raise ApiError(404, "not_found", "Object not found")
|
||||
except ApiError as exc:
|
||||
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
|
||||
except CacheUnavailableError:
|
||||
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
|
||||
except IdempotencyConflictError:
|
||||
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
|
||||
except AttachmentNotReadyError:
|
||||
self._send_error(409, "conflict", "Attachment is not ready", request_id)
|
||||
except ObjectMissingError:
|
||||
LOGGER.error("attachment object missing for request %s", request_id)
|
||||
self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id)
|
||||
except (BotoCoreError, ClientError):
|
||||
LOGGER.warning("object storage unavailable for request %s", request_id)
|
||||
self._send_error(503, "unavailable", "Object storage is unavailable", request_id)
|
||||
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
|
||||
self._send_error(503, "unavailable", "Database is unavailable", request_id)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
self._send_error(409, "conflict", "Username already exists", request_id)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
self._send_error(400, "invalid_request", "Unknown member", request_id)
|
||||
except Exception as exc:
|
||||
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
|
||||
self._send_error(500, "internal_error", "Internal server error", request_id)
|
||||
finally:
|
||||
if acquired:
|
||||
REQUEST_GUARD.release()
|
||||
|
||||
def _read_json(self, request_id: str) -> Any:
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None:
|
||||
raise ApiError(400, "invalid_request", "Missing Content-Length")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size < 0:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size > MAX_REQUEST_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Request body too large")
|
||||
raw = self.rfile.read(size)
|
||||
if not raw:
|
||||
raise ApiError(400, "invalid_request", "Empty body")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
raise ApiError(400, "invalid_request", "Malformed JSON")
|
||||
|
||||
def _get_actor(self, request_id: str) -> Dict[str, Any]:
|
||||
header = self.headers.get("Authorization", "")
|
||||
scheme, separator, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not separator or not token or " " in token:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
user = self.store.session_user(token)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
return user
|
||||
|
||||
def _handle_register(self, request_id: str):
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
|
||||
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
|
||||
username = body["username"]
|
||||
display_name = body["display_name"]
|
||||
password = body["password"]
|
||||
if not isinstance(username, str) or not USER_RE.fullmatch(username):
|
||||
raise ApiError(400, "invalid_request", "Invalid username")
|
||||
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid display_name")
|
||||
if not isinstance(password, str) or not (12 <= len(password) <= 128):
|
||||
raise ApiError(400, "invalid_request", "Invalid password")
|
||||
user = self.store.create_user(username, display_name, hash_password(password))
|
||||
self._send_json(201, user, request_id)
|
||||
|
||||
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
|
||||
value = self.headers.get("Authorization", "")
|
||||
scheme, separator, encoded = value.partition(" ")
|
||||
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
if scheme.lower() != "basic" or not separator or not encoded:
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError):
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
return username, password
|
||||
|
||||
def _handle_create_session(self, request_id: str):
|
||||
username, password = self._basic_credentials(request_id)
|
||||
user = self.store.authenticate(username, password)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials",
|
||||
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
token, expires_at = self.store.create_session(user["id"])
|
||||
self._send_json(
|
||||
201,
|
||||
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
|
||||
request_id,
|
||||
)
|
||||
|
||||
def _handle_revoke_session(self, request_id: str):
|
||||
self._get_actor(request_id)
|
||||
token = self.headers["Authorization"].partition(" ")[2]
|
||||
if not self.store.revoke_session(token):
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
self.send_response(204)
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.end_headers()
|
||||
|
||||
def _handle_create_chat(self, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
|
||||
raise ApiError(400, "invalid_request", "Expected title and member_ids")
|
||||
title, member_ids = body["title"], body["member_ids"]
|
||||
if not isinstance(title, str) or not title.strip() or len(title) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid title")
|
||||
if (
|
||||
not isinstance(member_ids, list)
|
||||
or not 1 <= len(member_ids) <= 100
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
|
||||
or len(member_ids) != len(set(member_ids))
|
||||
or actor["id"] not in member_ids
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid member_ids")
|
||||
chat = self.store.create_chat(title, member_ids, actor["id"])
|
||||
self._send_json(201, chat, request_id)
|
||||
|
||||
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _handle_get_chat(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat = self.store.get_chat(chat_id, actor["id"])
|
||||
if chat is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(200, chat, request_id)
|
||||
|
||||
def _handle_create_message(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
idempotency_key = self.headers.get("Idempotency-Key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body:
|
||||
raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids")
|
||||
text = body["text"]
|
||||
attachment_ids = body.get("attachment_ids", [])
|
||||
if (
|
||||
not isinstance(text, str)
|
||||
or len(text) > 4000
|
||||
or not isinstance(attachment_ids, list)
|
||||
or len(attachment_ids) > 10
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids)
|
||||
or len(attachment_ids) != len(set(attachment_ids))
|
||||
or (not text.strip() and not attachment_ids)
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid message text")
|
||||
attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids]
|
||||
canonical_body = json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
message = self.store.create_idempotent_message(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
text,
|
||||
attachment_ids,
|
||||
idempotency_key,
|
||||
canonical_body,
|
||||
)
|
||||
if message is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(201, message, request_id)
|
||||
|
||||
def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]:
|
||||
if declared_type not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None or self.headers.get("Transfer-Encoding"):
|
||||
raise ApiError(400, "invalid_request", "A bounded Content-Length is required")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size <= 0:
|
||||
raise ApiError(415, "unsupported_media_type", "Image is empty")
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
self.close_connection = True
|
||||
remaining = min(size, MAX_UPLOAD_BYTES + 1)
|
||||
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
|
||||
while remaining:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
break
|
||||
self.connection.settimeout(min(15, remaining_time))
|
||||
try:
|
||||
chunk = self.rfile.read(min(64 * 1024, remaining))
|
||||
except (TimeoutError, OSError):
|
||||
LOGGER.info("stopped bounded drain of oversized upload")
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
remaining -= len(chunk)
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit")
|
||||
content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
|
||||
digest = hashlib.sha256()
|
||||
bytes_read = 0
|
||||
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
|
||||
try:
|
||||
while bytes_read < size:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
raise ApiError(408, "invalid_request", "Image upload timed out")
|
||||
self.connection.settimeout(min(15, remaining_time))
|
||||
chunk = self.rfile.read(min(64 * 1024, size - bytes_read))
|
||||
if not chunk:
|
||||
raise ApiError(400, "invalid_request", "Incomplete image body")
|
||||
content.write(chunk)
|
||||
digest.update(chunk)
|
||||
bytes_read += len(chunk)
|
||||
except (TimeoutError, OSError):
|
||||
content.close()
|
||||
raise ApiError(400, "invalid_request", "Incomplete image body") from None
|
||||
|
||||
try:
|
||||
content.seek(0)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(content) as image:
|
||||
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "")
|
||||
if actual_type is None or actual_type != declared_type:
|
||||
raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type")
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit")
|
||||
image.verify()
|
||||
content.seek(0)
|
||||
with IMAGE_DECODE_GUARD:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(content) as image:
|
||||
image.load()
|
||||
content.seek(0)
|
||||
return content, actual_type, size, digest.hexdigest()
|
||||
except ApiError:
|
||||
content.close()
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
content.close()
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error):
|
||||
content.close()
|
||||
raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None
|
||||
|
||||
@staticmethod
|
||||
def _valid_filename(filename: str) -> bool:
|
||||
return (
|
||||
1 <= len(filename) <= 255
|
||||
and not any(char in ("/", "\\", "\x00") for char in filename)
|
||||
and not any(unicodedata.category(char) == "Cc" for char in filename)
|
||||
)
|
||||
|
||||
def _handle_upload_attachment(
|
||||
self, chat_id: str, query: Dict[str, List[str]], request_id: str
|
||||
):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
if not self.store.can_access_chat(chat_id, actor["id"]):
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
filenames = query.get("filename", [])
|
||||
if len(filenames) != 1 or not self._valid_filename(filenames[0]):
|
||||
raise ApiError(400, "invalid_request", "Invalid filename")
|
||||
if not UPLOAD_GUARD.acquire(blocking=False):
|
||||
raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted")
|
||||
try:
|
||||
content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type())
|
||||
try:
|
||||
attachment = self.store.create_attachment(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
filenames[0],
|
||||
content_type,
|
||||
content,
|
||||
size,
|
||||
digest,
|
||||
)
|
||||
finally:
|
||||
content.close()
|
||||
finally:
|
||||
UPLOAD_GUARD.release()
|
||||
if attachment is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(202, attachment, request_id)
|
||||
|
||||
def _handle_get_attachment(self, attachment_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(attachment_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment id")
|
||||
attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"])
|
||||
if attachment is None:
|
||||
raise ApiError(404, "not_found", "Attachment not found")
|
||||
self._send_json(200, attachment, request_id)
|
||||
|
||||
def _handle_download_attachment(
|
||||
self, attachment_id: str, query: Dict[str, List[str]], request_id: str
|
||||
):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(attachment_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment id")
|
||||
variants = query.get("variant", ["original"])
|
||||
if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment variant")
|
||||
variant = variants[0]
|
||||
if variant != "original" and variant not in ("thumbnail", "crop", "optimized"):
|
||||
raise ApiError(404, "not_found", "Attachment variant not found")
|
||||
result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant)
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Attachment not found")
|
||||
content_type, content = result
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(content)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Cache-Control", "private, no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
items, next_cursor = result
|
||||
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _send_json(
|
||||
self,
|
||||
status: int,
|
||||
payload: Dict[str, Any],
|
||||
request_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
if headers:
|
||||
for key, value in headers.items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_error(
|
||||
self,
|
||||
status: int,
|
||||
code: str,
|
||||
message: str,
|
||||
request_id: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._send_json(
|
||||
status,
|
||||
{"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
request_id,
|
||||
extra_headers,
|
||||
)
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pika
|
||||
|
||||
QUEUE_NAME = "attachments.process.v1"
|
||||
DEAD_QUEUE_NAME = "attachments.process.dead"
|
||||
LEASE_NETWORK_BUDGET_SECONDS = 150
|
||||
|
||||
|
||||
def broker_url() -> str:
|
||||
value = os.getenv("BROKER_URL")
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv("BROKER_URL_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError("Set BROKER_URL or BROKER_URL_FILE")
|
||||
|
||||
|
||||
def connect() -> pika.BlockingConnection:
|
||||
parameters = pika.URLParameters(broker_url())
|
||||
parameters.connection_attempts = 3
|
||||
parameters.retry_delay = 1
|
||||
parameters.socket_timeout = 5
|
||||
parameters.blocked_connection_timeout = 10
|
||||
return pika.BlockingConnection(parameters)
|
||||
|
||||
|
||||
def declare_queues(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
|
||||
channel.queue_declare(
|
||||
queue=DEAD_QUEUE_NAME,
|
||||
durable=True,
|
||||
arguments={"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
|
||||
)
|
||||
channel.queue_declare(
|
||||
queue=QUEUE_NAME,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-queue-type": "quorum",
|
||||
"x-quorum-initial-group-size": 1,
|
||||
"x-dead-letter-exchange": "",
|
||||
"x-dead-letter-routing-key": DEAD_QUEUE_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def encode_job(job: dict) -> bytes:
|
||||
return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level="INFO")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
store.migrate()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator, Optional, Tuple, Union
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY
|
||||
from .errors import ObjectMissingError
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
region_name=S3_REGION,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
config=Config(
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
|
||||
def put(
|
||||
self,
|
||||
key: str,
|
||||
content: Union[bytes, BinaryIO],
|
||||
content_type: str,
|
||||
size_bytes: Optional[int] = None,
|
||||
) -> None:
|
||||
if size_bytes is None:
|
||||
size_bytes = len(content)
|
||||
self.client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentLength=size_bytes,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key: str, max_bytes: int) -> bytes:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code in ("NoSuchKey", "404", "NotFound"):
|
||||
raise ObjectMissingError from exc
|
||||
raise
|
||||
body = response["Body"]
|
||||
try:
|
||||
value = body.read(max_bytes + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if len(value) > max_bytes:
|
||||
raise RuntimeError("Stored object exceeds configured safety limit")
|
||||
return value
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||
|
||||
def ready(self) -> bool:
|
||||
self.client.head_bucket(Bucket=S3_BUCKET)
|
||||
return True
|
||||
|
||||
def objects(self) -> Iterator[Tuple[str, datetime]]:
|
||||
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET)
|
||||
for page in pages:
|
||||
for item in page.get("Contents", []):
|
||||
yield item["Key"], item["LastModified"]
|
||||
@@ -0,0 +1,4 @@
|
||||
PIPELINE_QUALITY = {
|
||||
1: {"jpeg": 85, "webp": 80},
|
||||
2: {"jpeg": 80, "webp": 75},
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
|
||||
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues, encode_job
|
||||
from .security import isoformat
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger.publisher")
|
||||
|
||||
|
||||
def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
|
||||
with store.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = store.claim_outbox(conn)
|
||||
if row is None:
|
||||
return False
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(row["job_id"]),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": row["pipeline_version"],
|
||||
"attempt": row["attempt"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=QUEUE_NAME,
|
||||
body=encode_job(payload),
|
||||
properties=pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=2,
|
||||
message_id=str(row["id"]),
|
||||
type="attachment.process.v1",
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s",
|
||||
(row["id"],),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
while True:
|
||||
connection = None
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
declare_queues(channel)
|
||||
channel.confirm_delivery()
|
||||
store.recover_expired_attachment_jobs(
|
||||
int(os.getenv("JOB_MAX_ATTEMPTS", "5")),
|
||||
int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS,
|
||||
)
|
||||
if not publish_pending(store, channel):
|
||||
time.sleep(1)
|
||||
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
|
||||
LOGGER.exception("outbox publishing cycle failed")
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection is not None and connection.is_open:
|
||||
connection.close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("messenger.reconcile")
|
||||
|
||||
|
||||
def reconcile(
|
||||
store: Store,
|
||||
grace_seconds: int,
|
||||
apply_changes: bool,
|
||||
restore_dir: Optional[Path] = None,
|
||||
):
|
||||
started = time.monotonic()
|
||||
metadata = store.reconciliation_snapshot()
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
found = set()
|
||||
orphans = []
|
||||
for key, last_modified in store.objects.objects():
|
||||
found.add(key)
|
||||
if key not in metadata and last_modified <= cutoff:
|
||||
orphans.append(key)
|
||||
|
||||
missing = [
|
||||
row for key, row in metadata.items()
|
||||
if key not in found and row.get("reconcile_type", "original") == "original"
|
||||
]
|
||||
restore_candidates = []
|
||||
if restore_dir is not None:
|
||||
root = restore_dir.resolve(strict=True)
|
||||
for row in missing:
|
||||
source = restore_dir / str(row["id"])
|
||||
try:
|
||||
resolved = source.resolve(strict=True)
|
||||
if (
|
||||
resolved.parent != root
|
||||
or not resolved.is_file()
|
||||
or resolved.stat().st_size != row["size_bytes"]
|
||||
):
|
||||
continue
|
||||
content = resolved.read_bytes()
|
||||
except OSError:
|
||||
continue
|
||||
if (
|
||||
len(content) == row["size_bytes"]
|
||||
and hashlib.sha256(content).hexdigest() == row["sha256"].strip()
|
||||
):
|
||||
restore_candidates.append((row, content))
|
||||
|
||||
restored = 0
|
||||
if apply_changes:
|
||||
for key in orphans:
|
||||
store.objects.delete(key)
|
||||
restorable_ids = set()
|
||||
for row, content in restore_candidates:
|
||||
store.objects.put(row["object_key"], content, row["content_type"])
|
||||
store.clear_missing_object(row["id"])
|
||||
restorable_ids.add(row["id"])
|
||||
restored += 1
|
||||
for row in missing:
|
||||
if row["id"] not in restorable_ids:
|
||||
store.record_missing_object(row["id"])
|
||||
for key, row in metadata.items():
|
||||
if key in found and row.get("reconcile_type", "original") == "original":
|
||||
store.clear_missing_object(row["id"])
|
||||
|
||||
unresolved_missing = len(missing) - restored
|
||||
return {
|
||||
"mode": "apply" if apply_changes else "dry-run",
|
||||
"grace_seconds": grace_seconds,
|
||||
"objects_seen": len(found),
|
||||
"metadata_seen": len({str(row["id"]) for row in metadata.values()}),
|
||||
"orphan_objects": len([key for key in found if key not in metadata]),
|
||||
"stale_orphan_candidates": len(orphans),
|
||||
"deleted_orphans": len(orphans) if apply_changes else 0,
|
||||
"missing_objects": len(missing),
|
||||
"unresolved_missing_objects": unresolved_missing,
|
||||
"restorable_missing_objects": len(restore_candidates),
|
||||
"restored_missing_objects": restored,
|
||||
"missing_incidents_recorded": len(missing) - restored if apply_changes else 0,
|
||||
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.")
|
||||
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||
parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects")
|
||||
parser.add_argument(
|
||||
"--restore-dir",
|
||||
type=Path,
|
||||
help="optional trusted backup directory with exact original bytes named by attachment UUID",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error("--grace-seconds must be at least 60")
|
||||
|
||||
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
if result["unresolved_missing_objects"]:
|
||||
LOGGER.error(
|
||||
"%s database records reference missing S3 objects; metadata was retained for manual recovery",
|
||||
result["unresolved_missing_objects"],
|
||||
)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Explicitly replay a failed attachment or start a newer pipeline version."
|
||||
)
|
||||
parser.add_argument("--attachment-id", required=True, type=uuid.UUID)
|
||||
parser.add_argument(
|
||||
"--pipeline-version",
|
||||
required=True,
|
||||
type=int,
|
||||
help="pipeline version implemented by the deployed worker",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.pipeline_version < 1:
|
||||
parser.error("--pipeline-version must be positive")
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def isoformat(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def valid_uuid(value: str) -> bool:
|
||||
try:
|
||||
uuid.UUID(value)
|
||||
return True
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
derived = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
|
||||
)
|
||||
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
|
||||
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
|
||||
return f"scrypt$32768$8$1${salt_text}${hash_text}"
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
scheme, n, r, p, salt_text, hash_text = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
|
||||
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(expected),
|
||||
maxmem=64 * 1024 * 1024,
|
||||
)
|
||||
return hmac.compare_digest(actual, expected)
|
||||
except (ValueError, TypeError, UnicodeError):
|
||||
return False
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.app import main
|
||||
else:
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+835
@@ -0,0 +1,835 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
|
||||
|
||||
import redis
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .config import (
|
||||
DB_LOCK_TIMEOUT_MS,
|
||||
DB_POOL_MAX_SIZE,
|
||||
DB_POOL_TIMEOUT_SECONDS,
|
||||
DB_STATEMENT_TIMEOUT_MS,
|
||||
IDEMPOTENCY_TTL_SECONDS,
|
||||
SESSION_TTL_SECONDS,
|
||||
)
|
||||
from .errors import (
|
||||
AttachmentNotReadyError,
|
||||
CacheUnavailableError,
|
||||
IdempotencyConflictError,
|
||||
ObjectMissingError,
|
||||
)
|
||||
from .object_store import ObjectStore
|
||||
from .pipeline_versions import PIPELINE_QUALITY
|
||||
from .security import isoformat, utc_now, valid_uuid, verify_password
|
||||
|
||||
LOGGER = logging.getLogger("messenger.store")
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
value = os.getenv("DATABASE_URL")
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv("DATABASE_URL_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
|
||||
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
conninfo=database_url(),
|
||||
min_size=1,
|
||||
max_size=DB_POOL_MAX_SIZE,
|
||||
timeout=DB_POOL_TIMEOUT_SECONDS,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 5,
|
||||
"options": (
|
||||
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
|
||||
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
|
||||
"-c idle_in_transaction_session_timeout=10000"
|
||||
),
|
||||
},
|
||||
open=False,
|
||||
)
|
||||
cache_url = os.getenv("CACHE_URL")
|
||||
self.cache = (
|
||||
redis.Redis.from_url(
|
||||
cache_url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=0.25,
|
||||
socket_timeout=0.25,
|
||||
retry_on_timeout=False,
|
||||
max_connections=16,
|
||||
)
|
||||
if cache_url
|
||||
else None
|
||||
)
|
||||
self.objects = ObjectStore()
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=10)
|
||||
|
||||
def close(self):
|
||||
if self.cache is not None:
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self) -> bool:
|
||||
with self.pool.connection(timeout=2) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
self.objects.ready()
|
||||
return True
|
||||
|
||||
def migrate(self):
|
||||
migrations = Path(__file__).resolve().parents[1] / "migrations"
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
|
||||
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for migration in sorted(migrations.glob("*.sql")):
|
||||
if migration.name in applied:
|
||||
continue
|
||||
conn.execute(migration.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
|
||||
|
||||
@staticmethod
|
||||
def user(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
"attachment_ids": attachment_ids or [],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
variants = row.get("variants") or []
|
||||
public_variants = [
|
||||
{
|
||||
name: variant[name]
|
||||
for name in ("name", "content_type", "size_bytes", "width", "height", "sha256")
|
||||
}
|
||||
for variant in variants
|
||||
]
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"].strip(),
|
||||
"status": row.get("status", "queued"),
|
||||
"variants": public_variants,
|
||||
"error_code": row.get("error_code"),
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
"updated_at": isoformat(row["updated_at"]),
|
||||
}
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (username, display_name, password_hash) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
|
||||
(username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return self.user(row)
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
return self.user(row) if row else None
|
||||
|
||||
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash "
|
||||
"FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if not row or not verify_password(password, row["password_hash"]):
|
||||
return None
|
||||
return self.user(row)
|
||||
|
||||
def create_session(self, user_id: str) -> Tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
token_hash = hashlib.sha256(token.encode("ascii")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token_hash, user_id, expires_at),
|
||||
)
|
||||
return token, expires_at
|
||||
|
||||
@staticmethod
|
||||
def _session_lock_key(token_hash: bytes) -> int:
|
||||
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
|
||||
|
||||
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
cache_key = "messenger:session:" + token_hash.hex()
|
||||
if self.cache is not None:
|
||||
try:
|
||||
cached = self.cache.get(cache_key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
row = conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
|
||||
"FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
"WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
user = self.user(row)
|
||||
if self.cache is not None:
|
||||
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
|
||||
if ttl_ms > 0:
|
||||
try:
|
||||
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
|
||||
except redis.RedisError:
|
||||
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
|
||||
return user
|
||||
|
||||
def revoke_session(self, token: str) -> bool:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
if self.cache is not None:
|
||||
try:
|
||||
self.cache.delete("messenger:session:" + token_hash.hex())
|
||||
except redis.RedisError as exc:
|
||||
raise CacheUnavailableError from exc
|
||||
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
|
||||
return result.rowcount == 1
|
||||
|
||||
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
|
||||
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
|
||||
raise ValueError("invalid members")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
|
||||
(title,),
|
||||
).fetchone()
|
||||
for member_id in member_ids:
|
||||
conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
(row["id"], member_id),
|
||||
)
|
||||
return self.chat(row, member_ids)
|
||||
|
||||
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members m ON m.chat_id = c.id "
|
||||
"WHERE c.id = %s AND m.user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
return self.chat(row, [str(item["user_id"]) for item in members])
|
||||
|
||||
def list_chats(
|
||||
self, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
||||
where = "own.user_id = %s"
|
||||
params: List[Any] = [user_id]
|
||||
if cursor is not None:
|
||||
try:
|
||||
raw = base64.b64decode(
|
||||
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
|
||||
altchars=b"-_",
|
||||
validate=True,
|
||||
)
|
||||
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
|
||||
if cursor_user != user_id or not valid_uuid(chat_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None or created_at.utcoffset() is None:
|
||||
raise ValueError("cursor timestamp must include a timezone")
|
||||
where += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params.extend((created_at, chat_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members own ON own.chat_id = c.id "
|
||||
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
result = []
|
||||
for row in page:
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return result, next_cursor
|
||||
|
||||
def create_idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
attachment_ids: List[str],
|
||||
idempotency_key: str,
|
||||
canonical_body: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
|
||||
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
|
||||
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
).fetchone()
|
||||
if not member:
|
||||
return None
|
||||
if attachment_ids:
|
||||
attachments = conn.execute(
|
||||
"SELECT id, status FROM attachments "
|
||||
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
|
||||
(chat_id, attachment_ids),
|
||||
).fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
return None
|
||||
if any(item["status"] == "failed" for item in attachments):
|
||||
raise AttachmentNotReadyError
|
||||
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
|
||||
existing = conn.execute(
|
||||
"SELECT request_hash, response, expires_at "
|
||||
"FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
|
||||
"FOR UPDATE",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if existing and existing["expires_at"] > utc_now():
|
||||
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
|
||||
raise IdempotencyConflictError
|
||||
return existing["response"]
|
||||
if existing:
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
)
|
||||
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (chat_id, sender_id, text) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
|
||||
(chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
for ordinal, attachment_id in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
|
||||
"VALUES (%s, %s, %s)",
|
||||
(row["id"], attachment_id, ordinal),
|
||||
)
|
||||
message = self.message(row, attachment_ids)
|
||||
conn.execute(
|
||||
"INSERT INTO idempotency_records "
|
||||
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
|
||||
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
|
||||
(
|
||||
sender_id,
|
||||
chat_id,
|
||||
idempotency_key,
|
||||
request_hash,
|
||||
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
|
||||
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
|
||||
),
|
||||
)
|
||||
return message
|
||||
|
||||
def list_messages(
|
||||
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
|
||||
with self.pool.connection() as conn:
|
||||
is_member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not is_member:
|
||||
return None
|
||||
where = "chat_id = %s"
|
||||
params: List[Any] = [chat_id]
|
||||
if cursor:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
|
||||
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
|
||||
if decoded_chat != chat_id or not valid_uuid(message_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
where += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend((created_at, message_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
ids_by_message = {}
|
||||
if rows:
|
||||
links = conn.execute(
|
||||
"SELECT message_id, attachment_id FROM message_attachments "
|
||||
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
|
||||
([row["id"] for row in rows],),
|
||||
).fetchall()
|
||||
for link in links:
|
||||
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
|
||||
|
||||
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone() is not None
|
||||
|
||||
def create_attachment(
|
||||
self,
|
||||
chat_id: str,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
content: BinaryIO,
|
||||
size_bytes: int,
|
||||
digest: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not self.can_access_chat(chat_id, owner_id):
|
||||
return None
|
||||
attachment_id = uuid.uuid4()
|
||||
job_id = uuid.uuid4()
|
||||
staging_key = f"staging/{attachment_id}"
|
||||
object_key = f"original/{attachment_id}"
|
||||
content.seek(0)
|
||||
self.objects.put(staging_key, content, content_type, size_bytes)
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"object_key, staging_key, job_id) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
|
||||
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, variants, error_code, created_at, updated_at",
|
||||
(
|
||||
attachment_id,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size_bytes,
|
||||
digest,
|
||||
object_key,
|
||||
staging_key,
|
||||
job_id,
|
||||
),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt) "
|
||||
"VALUES (%s, %s, %s, 1, 1)",
|
||||
(uuid.uuid4(), job_id, attachment_id),
|
||||
)
|
||||
return self.attachment(row)
|
||||
except Exception:
|
||||
try:
|
||||
self.objects.delete(staging_key)
|
||||
except Exception as cleanup_error:
|
||||
LOGGER.error(
|
||||
"attachment staging compensation failed for %s (%s)",
|
||||
attachment_id,
|
||||
type(cleanup_error).__name__,
|
||||
)
|
||||
raise
|
||||
|
||||
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
|
||||
"a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at "
|
||||
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = %s AND m.user_id = %s",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
return self.attachment(row) if row else None
|
||||
|
||||
def download_attachment(
|
||||
self, attachment_id: str, user_id: str, variant: str = "original"
|
||||
) -> Optional[Tuple[str, bytes]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants "
|
||||
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = %s AND m.user_id = %s",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
if row["status"] != "ready":
|
||||
raise AttachmentNotReadyError
|
||||
if variant == "original":
|
||||
key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"]
|
||||
else:
|
||||
match = next(
|
||||
(item for item in (row["variants"] or []) if item["name"] == variant),
|
||||
None,
|
||||
)
|
||||
if match is None:
|
||||
return None
|
||||
key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"]
|
||||
return content_type, self.objects.get(key, size_bytes)
|
||||
|
||||
def claim_attachment_job(
|
||||
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, "
|
||||
"object_key, content_type, size_bytes, sha256 "
|
||||
"FROM attachments WHERE id = %s FOR UPDATE",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
not row
|
||||
or str(row["job_id"]) != job_id
|
||||
or row["pipeline_version"] != pipeline_version
|
||||
or row["status"] in ("ready", "failed")
|
||||
or row["attempt"] != attempt
|
||||
):
|
||||
return None
|
||||
if row["status"] == "processing" and row["lease_until"] is not None:
|
||||
if row["lease_until"] > utc_now():
|
||||
return None
|
||||
return conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', "
|
||||
"lease_until = clock_timestamp() + (%s * interval '1 second'), "
|
||||
"updated_at = clock_timestamp() "
|
||||
"WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, "
|
||||
"size_bytes, sha256, attempt, pipeline_version",
|
||||
(lease_seconds, attachment_id),
|
||||
).fetchone()
|
||||
|
||||
def complete_attachment_job(
|
||||
self,
|
||||
attachment_id: str,
|
||||
job_id: str,
|
||||
pipeline_version: int,
|
||||
attempt: int,
|
||||
variants: List[Dict[str, Any]],
|
||||
) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
updated = conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, "
|
||||
"lease_until = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s "
|
||||
"AND status = 'processing' AND lease_until > clock_timestamp()",
|
||||
(
|
||||
json.dumps(variants, separators=(",", ":")),
|
||||
attachment_id,
|
||||
job_id,
|
||||
pipeline_version,
|
||||
attempt,
|
||||
),
|
||||
)
|
||||
return updated.rowcount == 1
|
||||
|
||||
def fail_attachment_job(
|
||||
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int,
|
||||
max_attempts: int, error_code: str, retry_delay_seconds: Optional[float],
|
||||
) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT status, attempt FROM attachments "
|
||||
"WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE",
|
||||
(attachment_id, job_id, pipeline_version),
|
||||
).fetchone()
|
||||
if not row or row["status"] != "processing" or row["attempt"] != attempt:
|
||||
return
|
||||
if retry_delay_seconds is not None and attempt < max_attempts:
|
||||
next_attempt = attempt + 1
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
|
||||
"error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(next_attempt, error_code, attachment_id),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt, available_at) "
|
||||
"VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second')) "
|
||||
"ON CONFLICT (job_id, attempt) DO NOTHING",
|
||||
(
|
||||
uuid.uuid4(), job_id, attachment_id, pipeline_version,
|
||||
next_attempt, retry_delay_seconds,
|
||||
),
|
||||
)
|
||||
else:
|
||||
final_code = "retry_exhausted" if attempt >= max_attempts else error_code
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, "
|
||||
"updated_at = clock_timestamp() WHERE id = %s",
|
||||
(final_code, attachment_id),
|
||||
)
|
||||
|
||||
def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]:
|
||||
if pipeline_version not in PIPELINE_QUALITY:
|
||||
raise ValueError("pipeline version is not implemented by this deployment")
|
||||
with self.pool.connection() as conn:
|
||||
current = conn.execute(
|
||||
"SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, "
|
||||
"content_type, size_bytes, sha256 FROM attachments WHERE id = %s",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if current is None:
|
||||
raise ValueError("attachment not found")
|
||||
if pipeline_version < current["pipeline_version"]:
|
||||
raise ValueError("pipeline version cannot move backwards")
|
||||
if pipeline_version == current["pipeline_version"] and current["status"] != "failed":
|
||||
raise ValueError("same-version replay is allowed only for failed attachments")
|
||||
if current["status"] not in ("queued", "processing", "ready", "failed"):
|
||||
raise ValueError("attachment cannot be replayed in its current state")
|
||||
|
||||
try:
|
||||
content = self.objects.get(current["object_key"], current["size_bytes"])
|
||||
except ObjectMissingError as object_error:
|
||||
try:
|
||||
content = self.objects.get(current["staging_key"], current["size_bytes"])
|
||||
except ObjectMissingError:
|
||||
raise object_error
|
||||
if (
|
||||
len(content) != current["size_bytes"]
|
||||
or hashlib.sha256(content).hexdigest() != current["sha256"].strip()
|
||||
):
|
||||
raise ValueError("stored original does not match attachment metadata")
|
||||
|
||||
prior_job_id = str(current["job_id"])
|
||||
next_job_id = uuid.uuid4()
|
||||
staging_key = f"staging/{attachment_id}/{next_job_id}"
|
||||
self.objects.put(staging_key, content, current["content_type"], len(content))
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
locked = conn.execute(
|
||||
"SELECT job_id, pipeline_version, attempt, status FROM attachments "
|
||||
"WHERE id = %s FOR UPDATE",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
locked is None
|
||||
or str(locked["job_id"]) != prior_job_id
|
||||
or locked["pipeline_version"] != current["pipeline_version"]
|
||||
or locked["attempt"] != current["attempt"]
|
||||
or locked["status"] != current["status"]
|
||||
):
|
||||
raise RuntimeError("attachment changed while preparing replay")
|
||||
conn.execute(
|
||||
"UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, "
|
||||
"status = 'queued', staging_key = %s, variants = '[]'::jsonb, "
|
||||
"error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s",
|
||||
(next_job_id, pipeline_version, staging_key, attachment_id),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt) "
|
||||
"VALUES (%s, %s, %s, %s, 1)",
|
||||
(uuid.uuid4(), next_job_id, attachment_id, pipeline_version),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_pipeline_events "
|
||||
"(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) "
|
||||
"VALUES (%s, %s, %s, %s, %s, 'replay')",
|
||||
(
|
||||
attachment_id,
|
||||
prior_job_id,
|
||||
next_job_id,
|
||||
current["pipeline_version"],
|
||||
pipeline_version,
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
self.objects.delete(staging_key)
|
||||
except Exception as cleanup_error:
|
||||
LOGGER.error(
|
||||
"replay staging cleanup failed for %s (%s)",
|
||||
attachment_id,
|
||||
type(cleanup_error).__name__,
|
||||
)
|
||||
raise
|
||||
return {
|
||||
"attachment_id": str(attachment_id),
|
||||
"previous_job_id": prior_job_id,
|
||||
"job_id": str(next_job_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": 1,
|
||||
"status": "queued",
|
||||
}
|
||||
|
||||
def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"SELECT id, job_id, pipeline_version, attempt FROM attachments "
|
||||
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
|
||||
"ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
if row["attempt"] >= max_attempts:
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', "
|
||||
"lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(row["id"],),
|
||||
)
|
||||
continue
|
||||
next_attempt = row["attempt"] + 1
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
|
||||
"error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s",
|
||||
(next_attempt, row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt) "
|
||||
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING",
|
||||
(uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt),
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def claim_outbox(self, conn):
|
||||
return conn.execute(
|
||||
"SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, "
|
||||
"a.created_at FROM attachment_outbox o "
|
||||
"JOIN attachments a ON a.id = o.attachment_id "
|
||||
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
|
||||
"ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1"
|
||||
).fetchone()
|
||||
|
||||
def reconciliation_snapshot(self):
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, "
|
||||
"size_bytes, sha256, content_type FROM attachments"
|
||||
).fetchall()
|
||||
expected = {}
|
||||
for row in rows:
|
||||
if row["status"] == "ready":
|
||||
expected[row["object_key"]] = {**row, "reconcile_type": "original"}
|
||||
for variant in row["variants"] or []:
|
||||
expected[variant["object_key"]] = {
|
||||
**row,
|
||||
"object_key": variant["object_key"],
|
||||
"size_bytes": variant["size_bytes"],
|
||||
"sha256": variant["sha256"],
|
||||
"content_type": variant["content_type"],
|
||||
"reconcile_type": "variant",
|
||||
}
|
||||
elif row["status"] in ("queued", "processing"):
|
||||
expected[row["staging_key"]] = {**row, "reconcile_type": "staging"}
|
||||
expected[row["object_key"]] = {**row, "reconcile_type": "inflight"}
|
||||
for name, extension in (
|
||||
("thumbnail", "jpg"),
|
||||
("crop", "jpg"),
|
||||
("optimized", "webp"),
|
||||
):
|
||||
key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}"
|
||||
expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"}
|
||||
return expected
|
||||
|
||||
def record_missing_object(self, attachment_id: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
|
||||
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
|
||||
"DO UPDATE SET last_seen_at = clock_timestamp()",
|
||||
(attachment_id,),
|
||||
)
|
||||
|
||||
def clear_missing_object(self, attachment_id: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM reconciliation_incidents "
|
||||
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
|
||||
(attachment_id,),
|
||||
)
|
||||
+341
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
import warnings
|
||||
import zlib
|
||||
from typing import Callable, Optional
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from .config import MAX_IMAGE_PIXELS
|
||||
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues
|
||||
from .pipeline_versions import PIPELINE_QUALITY
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger.worker")
|
||||
|
||||
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
|
||||
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
|
||||
|
||||
|
||||
class InvalidAttachment(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedPipeline(InvalidAttachment):
|
||||
pass
|
||||
|
||||
|
||||
def _rgb_image(content: bytes) -> Image.Image:
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(io.BytesIO(content)) as source:
|
||||
if source.format not in ("PNG", "JPEG"):
|
||||
raise InvalidAttachment("unsupported_image")
|
||||
width, height = source.size
|
||||
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
|
||||
raise InvalidAttachment("image_too_large")
|
||||
source.load()
|
||||
image = ImageOps.exif_transpose(source)
|
||||
try:
|
||||
if "A" in image.getbands() or "transparency" in image.info:
|
||||
with image.convert("RGBA") as rgba:
|
||||
background = Image.new("RGB", rgba.size, "white")
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
return image.convert("RGB")
|
||||
finally:
|
||||
if image is not source:
|
||||
image.close()
|
||||
except InvalidAttachment:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError,
|
||||
UnidentifiedImageError, ValueError, zlib.error) as exc:
|
||||
raise InvalidAttachment("invalid_image") from exc
|
||||
|
||||
|
||||
def _encode(image: Image.Image, image_format: str, **options) -> bytes:
|
||||
output = io.BytesIO()
|
||||
image.save(output, format=image_format, **options)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def _resize_within(image: Image.Image, bounds: tuple[int, int]) -> Image.Image:
|
||||
width, height = image.size
|
||||
scale = min(1.0, bounds[0] / width, bounds[1] / height)
|
||||
size = (max(1, int(width * scale)), max(1, int(height * scale)))
|
||||
if size == image.size:
|
||||
return image.copy()
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1):
|
||||
quality = PIPELINE_QUALITY.get(pipeline_version)
|
||||
if quality is None:
|
||||
raise UnsupportedPipeline("unsupported_pipeline_version")
|
||||
image = _rgb_image(content)
|
||||
try:
|
||||
variants = []
|
||||
thumbnail = _resize_within(image, (256, 256))
|
||||
try:
|
||||
thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True)
|
||||
variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes))
|
||||
finally:
|
||||
thumbnail.close()
|
||||
|
||||
crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
|
||||
try:
|
||||
crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True)
|
||||
variants.append(("crop", "image/jpeg", 128, 128, crop_bytes))
|
||||
finally:
|
||||
crop.close()
|
||||
|
||||
optimized = _resize_within(image, (1280, 1280))
|
||||
try:
|
||||
optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6)
|
||||
variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes))
|
||||
finally:
|
||||
optimized.close()
|
||||
|
||||
result = []
|
||||
for name, content_type, width, height, data in variants:
|
||||
extension = "webp" if content_type == "image/webp" else "jpg"
|
||||
key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}"
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(data),
|
||||
"width": width,
|
||||
"height": height,
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"object_key": key,
|
||||
"content": data,
|
||||
}
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
image.close()
|
||||
|
||||
|
||||
class ProcessingTimeout(TimeoutError):
|
||||
pass
|
||||
|
||||
|
||||
def _variant_process(connection, attachment_id, content, pipeline_version, builder):
|
||||
try:
|
||||
connection.send(("ok", builder(attachment_id, content, pipeline_version)))
|
||||
except InvalidAttachment as exc:
|
||||
connection.send(("invalid", str(exc)))
|
||||
except Exception as exc:
|
||||
connection.send(("error", type(exc).__name__))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _build_variants_bounded(
|
||||
attachment_id: str,
|
||||
content: bytes,
|
||||
pipeline_version: int,
|
||||
timeout_seconds: int,
|
||||
builder=build_variants,
|
||||
):
|
||||
if timeout_seconds < 1:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
context = multiprocessing.get_context("spawn")
|
||||
receiver, sender = context.Pipe(duplex=False)
|
||||
process = context.Process(
|
||||
target=_variant_process,
|
||||
args=(sender, attachment_id, content, pipeline_version, builder),
|
||||
)
|
||||
process.start()
|
||||
sender.close()
|
||||
try:
|
||||
if not receiver.poll(timeout_seconds):
|
||||
process.terminate()
|
||||
process.join(timeout=2)
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
raise ProcessingTimeout("image processing exceeded its time budget")
|
||||
try:
|
||||
result, value = receiver.recv()
|
||||
except EOFError as exc:
|
||||
raise RuntimeError("image processing process exited without a result") from exc
|
||||
process.join(timeout=2)
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join()
|
||||
if result == "invalid":
|
||||
raise InvalidAttachment(value)
|
||||
if result == "error":
|
||||
raise RuntimeError(f"image processing failed in subprocess ({value})")
|
||||
return value
|
||||
finally:
|
||||
receiver.close()
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join()
|
||||
|
||||
|
||||
def _job(payload: bytes):
|
||||
value = json.loads(payload)
|
||||
required = {
|
||||
"schema_version", "job_id", "type", "attachment_id",
|
||||
"pipeline_version", "attempt", "created_at",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}):
|
||||
raise ValueError("invalid job fields")
|
||||
if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1":
|
||||
raise ValueError("invalid job schema")
|
||||
uuid.UUID(value["job_id"])
|
||||
uuid.UUID(value["attachment_id"])
|
||||
if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1:
|
||||
raise ValueError("invalid pipeline version")
|
||||
if not isinstance(value["attempt"], int) or value["attempt"] < 1:
|
||||
raise ValueError("invalid job attempt")
|
||||
return value
|
||||
|
||||
|
||||
def _process(
|
||||
store: Store,
|
||||
job: dict,
|
||||
fault_injector: Optional[Callable[[str], None]] = None,
|
||||
) -> None:
|
||||
def inject(point: str) -> None:
|
||||
if fault_injector is not None:
|
||||
fault_injector(point)
|
||||
|
||||
attachment_id = job["attachment_id"]
|
||||
row = store.claim_attachment_job(
|
||||
attachment_id,
|
||||
job["job_id"],
|
||||
job["pipeline_version"],
|
||||
job["attempt"],
|
||||
JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS,
|
||||
)
|
||||
if row is None:
|
||||
LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"])
|
||||
return
|
||||
|
||||
try:
|
||||
inject("after_claim")
|
||||
content = store.objects.get(row["staging_key"], row["size_bytes"])
|
||||
if hashlib.sha256(content).hexdigest() != row["sha256"].strip():
|
||||
raise InvalidAttachment("checksum_mismatch")
|
||||
variants = _build_variants_bounded(
|
||||
attachment_id,
|
||||
content,
|
||||
job["pipeline_version"],
|
||||
JOB_TIMEOUT_SECONDS,
|
||||
)
|
||||
inject("before_original_put")
|
||||
store.objects.put(row["object_key"], content, row["content_type"], len(content))
|
||||
inject("after_original_put")
|
||||
persisted = []
|
||||
for variant in variants:
|
||||
data = variant.pop("content")
|
||||
inject(f"before_variant_put:{variant['name']}")
|
||||
store.objects.put(variant["object_key"], data, variant["content_type"], len(data))
|
||||
inject(f"after_variant_put:{variant['name']}")
|
||||
persisted.append(variant)
|
||||
inject("before_commit")
|
||||
if not store.complete_attachment_job(
|
||||
attachment_id,
|
||||
job["job_id"],
|
||||
job["pipeline_version"],
|
||||
job["attempt"],
|
||||
persisted,
|
||||
):
|
||||
LOGGER.warning("job lease was superseded before completion: %s", job["job_id"])
|
||||
return
|
||||
inject("after_commit")
|
||||
try:
|
||||
store.objects.delete(row["staging_key"])
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id)
|
||||
LOGGER.info("attachment processing completed: %s", attachment_id)
|
||||
except InvalidAttachment as exc:
|
||||
store.fail_attachment_job(
|
||||
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
|
||||
MAX_ATTEMPTS, str(exc), None,
|
||||
)
|
||||
try:
|
||||
store.objects.delete(row["staging_key"])
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id)
|
||||
LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc)
|
||||
except Exception as exc:
|
||||
delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25)
|
||||
store.fail_attachment_job(
|
||||
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
|
||||
MAX_ATTEMPTS, "processing_failed", delay,
|
||||
)
|
||||
LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__)
|
||||
|
||||
|
||||
def handle_delivery(
|
||||
store: Store,
|
||||
channel: pika.adapters.blocking_connection.BlockingChannel,
|
||||
method,
|
||||
body: bytes,
|
||||
fault_injector: Optional[Callable[[str], None]] = None,
|
||||
) -> None:
|
||||
try:
|
||||
job = _job(body)
|
||||
except (ValueError, TypeError, json.JSONDecodeError):
|
||||
LOGGER.exception("invalid broker job rejected")
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
|
||||
return
|
||||
_process(store, job, fault_injector)
|
||||
if fault_injector is not None:
|
||||
fault_injector("before_ack")
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1:
|
||||
raise RuntimeError("Worker limits must be positive")
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
|
||||
store = Store()
|
||||
store.open()
|
||||
connection = None
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
declare_queues(channel)
|
||||
channel.basic_qos(prefetch_count=WORKER_CONCURRENCY)
|
||||
|
||||
def on_message(ch, method, properties, body):
|
||||
handle_delivery(store, ch, method, body)
|
||||
|
||||
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False)
|
||||
LOGGER.info("worker consuming durable attachment queue")
|
||||
channel.start_consuming()
|
||||
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
|
||||
LOGGER.exception("worker connection failed")
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection is not None and connection.is_open:
|
||||
connection.close()
|
||||
connection = None
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative attachment and cross-chat authorization checks against a running lab4 API."""
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202):
|
||||
return client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""),
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=data,
|
||||
headers={"Content-Type": content_type},
|
||||
schema="Attachment" if expected in (201, 202) else None,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Attachment upload, size, MIME, and chat authorization checks")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
other_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
print("Checking accepted fixture upload.", flush=True)
|
||||
attachment = upload(client, alice, chat["id"], image)["json"]
|
||||
|
||||
print("Checking damaged and spoofed image rejection.", flush=True)
|
||||
upload(client, alice, chat["id"], b"not an image", expected=415)
|
||||
upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415)
|
||||
print("Checking byte and pixel limits.", flush=True)
|
||||
upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413)
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
def chunk(kind, payload):
|
||||
return (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30"))
|
||||
bad_idat[-3] ^= 0xFF
|
||||
damaged_idat = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", bytes(bad_idat))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
upload(client, alice, chat["id"], damaged_idat, expected=415)
|
||||
|
||||
too_many_pixels = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(b""))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
upload(client, alice, chat["id"], too_many_pixels, expected=413)
|
||||
print("Checking filename and cross-chat authorization.", flush=True)
|
||||
upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400)
|
||||
upload(client, eve, chat["id"], image, expected=404)
|
||||
client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + other_chat["id"] + "/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
400,
|
||||
actor=bob,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
body={"text": ""},
|
||||
)
|
||||
print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a concurrent idempotency-key race through the load balancer."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Concurrent shared-idempotency integration check")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = "race-" + uuid.uuid4().hex
|
||||
text = "concurrent-" + uuid.uuid4().hex
|
||||
count = 12
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def send_once(_):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=10)
|
||||
return worker.message(alice, chat, text, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(send_once, range(count)))
|
||||
|
||||
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
|
||||
messages = [response["json"] for response in responses]
|
||||
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
|
||||
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
matches = [message for message in page["items"] if message["text"] == text]
|
||||
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
|
||||
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify an upload committed while the publisher is stopped is processed after restart."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify committed attachment outbox intent survives publisher downtime")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
compose("stop", "publisher")
|
||||
try:
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload was not accepted into queued state")
|
||||
finally:
|
||||
compose("start", "publisher")
|
||||
|
||||
ready = client.wait_ready(alice, attachment)
|
||||
stored_original = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
thumbnail = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
variant = next(item for item in ready["variants"] if item["name"] == "thumbnail")
|
||||
check(stored_original == original, "recovered job changed original bytes")
|
||||
check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"accepted_status_before_publisher_restart": attachment["status"],
|
||||
"final_status": ready["status"],
|
||||
"attachment_id": attachment["id"],
|
||||
"original_sha256": hashlib.sha256(stored_original).hexdigest(),
|
||||
"thumbnail_sha256": variant["sha256"],
|
||||
"recovered_without_manual_republish": True,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import pika
|
||||
|
||||
from src.publisher import publish_pending
|
||||
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
|
||||
|
||||
|
||||
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
|
||||
JOB_ID = "00000000-0000-4000-8000-000000000002"
|
||||
|
||||
|
||||
class SimulatedCrash(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
def __init__(self, content):
|
||||
self.data = {"staging/" + ATTACHMENT_ID: content}
|
||||
self.puts = []
|
||||
|
||||
def get(self, key, max_bytes):
|
||||
return self.data[key]
|
||||
|
||||
def put(self, key, content, content_type, size_bytes=None):
|
||||
value = content if isinstance(content, bytes) else content.read()
|
||||
assert len(value) == size_bytes
|
||||
self.data[key] = value
|
||||
self.puts.append((key, hashlib.sha256(value).hexdigest()))
|
||||
|
||||
def delete(self, key):
|
||||
self.data.pop(key, None)
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self, content):
|
||||
self.objects = FakeObjects(content)
|
||||
self.row = {
|
||||
"id": ATTACHMENT_ID,
|
||||
"job_id": JOB_ID,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"status": "queued",
|
||||
"lease_until": 0.0,
|
||||
"staging_key": "staging/" + ATTACHMENT_ID,
|
||||
"object_key": "original/" + ATTACHMENT_ID,
|
||||
"content_type": "image/png",
|
||||
"size_bytes": len(content),
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"variants": [],
|
||||
}
|
||||
|
||||
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
|
||||
row = self.row
|
||||
if (
|
||||
attachment_id != row["id"]
|
||||
or job_id != row["job_id"]
|
||||
or version != row["pipeline_version"]
|
||||
or attempt != row["attempt"]
|
||||
or row["status"] in ("ready", "failed")
|
||||
):
|
||||
return None
|
||||
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
|
||||
return None
|
||||
row["status"] = "processing"
|
||||
row["lease_until"] = time.monotonic() + lease_seconds
|
||||
return dict(row)
|
||||
|
||||
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
|
||||
row = self.row
|
||||
if (
|
||||
attachment_id != row["id"]
|
||||
or job_id != row["job_id"]
|
||||
or version != row["pipeline_version"]
|
||||
or attempt != row["attempt"]
|
||||
or row["status"] != "processing"
|
||||
or row["lease_until"] <= time.monotonic()
|
||||
):
|
||||
return False
|
||||
row["status"] = "ready"
|
||||
row["lease_until"] = 0.0
|
||||
row["variants"] = variants
|
||||
return True
|
||||
|
||||
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
|
||||
row = self.row
|
||||
if (
|
||||
row["id"] == attachment_id
|
||||
and row["job_id"] == job_id
|
||||
and row["pipeline_version"] == version
|
||||
and row["attempt"] == attempt
|
||||
and row["status"] == "processing"
|
||||
):
|
||||
row["status"] = "failed" if delay is None else "queued"
|
||||
row["lease_until"] = 0.0
|
||||
row["error_code"] = error_code
|
||||
if delay is not None:
|
||||
row["attempt"] += 1
|
||||
|
||||
def recover_expired(self):
|
||||
if self.row["status"] == "processing":
|
||||
self.row["attempt"] += 1
|
||||
self.row["status"] = "queued"
|
||||
self.row["lease_until"] = 0.0
|
||||
|
||||
def replay(self, version):
|
||||
row = self.row
|
||||
previous_job = row["job_id"]
|
||||
row["job_id"] = str(uuid.uuid4())
|
||||
row["pipeline_version"] = version
|
||||
row["attempt"] = 1
|
||||
row["status"] = "queued"
|
||||
row["lease_until"] = 0.0
|
||||
row["variants"] = []
|
||||
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
|
||||
self.objects.put(
|
||||
row["staging_key"],
|
||||
self.objects.data[row["object_key"]],
|
||||
row["content_type"],
|
||||
row["size_bytes"],
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"job_id": row["job_id"],
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": ATTACHMENT_ID,
|
||||
"pipeline_version": version,
|
||||
"attempt": 1,
|
||||
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
"previous_job_id": previous_job,
|
||||
}
|
||||
|
||||
|
||||
class FakeChannel:
|
||||
def __init__(self):
|
||||
self.acked = []
|
||||
self.nacked = []
|
||||
|
||||
def basic_ack(self, delivery_tag):
|
||||
self.acked.append(delivery_tag)
|
||||
|
||||
def basic_nack(self, delivery_tag, requeue):
|
||||
self.nacked.append((delivery_tag, requeue))
|
||||
|
||||
|
||||
class FakeMethod:
|
||||
def __init__(self, delivery_tag):
|
||||
self.delivery_tag = delivery_tag
|
||||
|
||||
|
||||
def job_body(job_id=JOB_ID, version=1, attempt=1):
|
||||
return json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": ATTACHMENT_ID,
|
||||
"pipeline_version": version,
|
||||
"attempt": attempt,
|
||||
"created_at": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def crash_at(expected):
|
||||
def inject(point):
|
||||
if point == expected:
|
||||
raise SimulatedCrash(point)
|
||||
|
||||
return inject
|
||||
|
||||
|
||||
def slow_builder(attachment_id, content, pipeline_version):
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def test_duplicate_and_ack_loss(content):
|
||||
store = FakeStore(content)
|
||||
channel = FakeChannel()
|
||||
try:
|
||||
handle_delivery(
|
||||
store, channel, FakeMethod(1), job_body(),
|
||||
fault_injector=crash_at("before_ack"),
|
||||
)
|
||||
raise AssertionError("fault injection did not crash before ack")
|
||||
except SimulatedCrash:
|
||||
pass
|
||||
assert store.row["status"] == "ready"
|
||||
first_write_count = len(store.objects.puts)
|
||||
assert channel.acked == []
|
||||
|
||||
handle_delivery(store, channel, FakeMethod(2), job_body())
|
||||
assert channel.acked == [2]
|
||||
assert len(store.objects.puts) == first_write_count
|
||||
assert store.row["status"] == "ready"
|
||||
return {
|
||||
"scenario": "duplicate_delivery_after_commit_before_ack",
|
||||
"deliveries": 2,
|
||||
"effective_object_writes": first_write_count,
|
||||
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
|
||||
}
|
||||
|
||||
|
||||
def test_crash_before_and_after_side_effect(content, crash_point):
|
||||
store = FakeStore(content)
|
||||
channel = FakeChannel()
|
||||
try:
|
||||
handle_delivery(
|
||||
store, channel, FakeMethod(3), job_body(),
|
||||
fault_injector=crash_at(crash_point),
|
||||
)
|
||||
raise AssertionError(f"fault injection did not crash at {crash_point}")
|
||||
except SimulatedCrash:
|
||||
pass
|
||||
assert channel.acked == []
|
||||
store.recover_expired()
|
||||
retry = job_body(attempt=2)
|
||||
handle_delivery(store, channel, FakeMethod(4), retry)
|
||||
assert store.row["status"] == "ready"
|
||||
assert store.row["attempt"] == 2
|
||||
assert channel.acked == [4]
|
||||
assert store.objects.data["original/" + ATTACHMENT_ID] == content
|
||||
return {
|
||||
"scenario": "crash_" + crash_point,
|
||||
"attempts": store.row["attempt"],
|
||||
"terminal_status": store.row["status"],
|
||||
}
|
||||
|
||||
|
||||
def test_reordered_attempts(content):
|
||||
store = FakeStore(content)
|
||||
store.row["attempt"] = 2
|
||||
channel = FakeChannel()
|
||||
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
|
||||
writes_after_current = len(store.objects.puts)
|
||||
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
|
||||
assert store.row["status"] == "ready"
|
||||
assert len(store.objects.puts) == writes_after_current
|
||||
assert channel.acked == [5, 6]
|
||||
return {
|
||||
"scenario": "out_of_order_attempt_delivery",
|
||||
"current_attempt": store.row["attempt"],
|
||||
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
|
||||
}
|
||||
|
||||
|
||||
def test_stale_pipeline_cannot_publish_newer_result(content):
|
||||
store = FakeStore(content)
|
||||
channel = FakeChannel()
|
||||
replayed = []
|
||||
|
||||
def supersede_after_first_variant(point):
|
||||
if point == "after_variant_put:thumbnail" and not replayed:
|
||||
replayed.append(store.replay(2))
|
||||
|
||||
handle_delivery(
|
||||
store, channel, FakeMethod(7), job_body(),
|
||||
fault_injector=supersede_after_first_variant,
|
||||
)
|
||||
assert store.row["status"] == "queued"
|
||||
assert store.row["pipeline_version"] == 2
|
||||
assert channel.acked == [7]
|
||||
|
||||
current_job = replayed[0]
|
||||
handle_delivery(
|
||||
store,
|
||||
channel,
|
||||
FakeMethod(8),
|
||||
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
|
||||
)
|
||||
assert store.row["status"] == "ready"
|
||||
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
|
||||
assert channel.acked == [7, 8]
|
||||
return {
|
||||
"scenario": "stale_pipeline_after_newer_version",
|
||||
"current_pipeline_version": store.row["pipeline_version"],
|
||||
"published_variant_versions": sorted(
|
||||
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class FakeTransaction:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
|
||||
def __enter__(self):
|
||||
self.published = False
|
||||
self.store.current_transaction = self
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
if exc_type is None and self.published:
|
||||
self.store.outbox_published = True
|
||||
self.store.current_transaction = None
|
||||
return False
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
|
||||
def transaction(self):
|
||||
return FakeTransaction(self.store)
|
||||
|
||||
def execute(self, query, params):
|
||||
assert "UPDATE attachment_outbox" in query
|
||||
assert params == ("outbox-row",)
|
||||
self.store.current_transaction.published = True
|
||||
|
||||
|
||||
class FakePool:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
|
||||
def connection(self):
|
||||
connection = FakeConnection(self.store)
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self_inner):
|
||||
return connection
|
||||
|
||||
def __exit__(self_inner, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
return ConnectionContext()
|
||||
|
||||
|
||||
class FakeOutboxStore:
|
||||
def __init__(self):
|
||||
self.pool = FakePool(self)
|
||||
self.outbox_published = False
|
||||
self.current_transaction = None
|
||||
|
||||
def claim_outbox(self, connection):
|
||||
if self.outbox_published:
|
||||
return None
|
||||
return {
|
||||
"id": "outbox-row",
|
||||
"job_id": JOB_ID,
|
||||
"attachment_id": ATTACHMENT_ID,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
}
|
||||
|
||||
|
||||
class AmbiguousConfirmChannel:
|
||||
def __init__(self):
|
||||
self.messages = []
|
||||
self.fail_once = True
|
||||
|
||||
def basic_publish(self, **message):
|
||||
self.messages.append(message)
|
||||
if self.fail_once:
|
||||
self.fail_once = False
|
||||
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
|
||||
|
||||
|
||||
class FailBeforePublishChannel:
|
||||
def __init__(self):
|
||||
self.messages = []
|
||||
self.fail_once = True
|
||||
|
||||
def basic_publish(self, **message):
|
||||
if self.fail_once:
|
||||
self.fail_once = False
|
||||
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
|
||||
self.messages.append(message)
|
||||
|
||||
|
||||
def test_committed_outbox_survives_publisher_crash():
|
||||
store = FakeOutboxStore()
|
||||
channel = FailBeforePublishChannel()
|
||||
try:
|
||||
publish_pending(store, channel)
|
||||
raise AssertionError("publisher crash was not injected")
|
||||
except pika.exceptions.AMQPConnectionError:
|
||||
pass
|
||||
assert not store.outbox_published
|
||||
assert channel.messages == []
|
||||
assert publish_pending(store, channel)
|
||||
assert store.outbox_published
|
||||
assert len(channel.messages) == 1
|
||||
return {
|
||||
"scenario": "publisher_crash_before_publish",
|
||||
"published_after_restart": len(channel.messages),
|
||||
"outbox_recovered_after_restart": store.outbox_published,
|
||||
}
|
||||
|
||||
|
||||
def test_outbox_confirm_gap_republishes_same_job():
|
||||
store = FakeOutboxStore()
|
||||
channel = AmbiguousConfirmChannel()
|
||||
try:
|
||||
publish_pending(store, channel)
|
||||
raise AssertionError("publisher confirm fault was not injected")
|
||||
except pika.exceptions.AMQPConnectionError:
|
||||
pass
|
||||
assert not store.outbox_published
|
||||
assert publish_pending(store, channel)
|
||||
assert store.outbox_published
|
||||
assert len(channel.messages) == 2
|
||||
first, second = channel.messages
|
||||
assert first["body"] == second["body"]
|
||||
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
|
||||
return {
|
||||
"scenario": "publish_confirm_lost_after_broker_accept",
|
||||
"deliveries": len(channel.messages),
|
||||
"same_message_id": True,
|
||||
}
|
||||
|
||||
|
||||
def test_image_processing_has_a_hard_timeout():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
_build_variants_bounded(
|
||||
ATTACHMENT_ID,
|
||||
b"bounded timeout test",
|
||||
1,
|
||||
timeout_seconds=1,
|
||||
builder=slow_builder,
|
||||
)
|
||||
raise AssertionError("slow transform exceeded its time budget without termination")
|
||||
except ProcessingTimeout:
|
||||
pass
|
||||
elapsed = time.monotonic() - started
|
||||
assert elapsed < 4
|
||||
return {
|
||||
"scenario": "transform_timeout_terminated",
|
||||
"configured_timeout_seconds": 1,
|
||||
"observed_seconds": round(elapsed, 3),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
results = [
|
||||
test_duplicate_and_ack_loss(content),
|
||||
test_crash_before_and_after_side_effect(content, "after_claim"),
|
||||
test_crash_before_and_after_side_effect(content, "after_original_put"),
|
||||
test_reordered_attempts(content),
|
||||
test_stale_pipeline_cannot_publish_newer_result(content),
|
||||
test_committed_outbox_survives_publisher_crash(),
|
||||
test_outbox_confirm_gap_republishes_same_job(),
|
||||
test_image_processing_has_a_hard_timeout(),
|
||||
]
|
||||
report = {
|
||||
"result": "PASS",
|
||||
"track": "2",
|
||||
"fixture": "tests/fixtures/sample.png",
|
||||
"scenarios": results,
|
||||
"invariants": {
|
||||
"confirmed_work_eventually_ready": True,
|
||||
"duplicate_effects_create_no_second_variant_set": True,
|
||||
"stale_attempt_writes_no_effects": True,
|
||||
"stale_pipeline_cannot_publish_current_metadata": True,
|
||||
"ambiguous_confirm_republishes_stable_job": True,
|
||||
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
|
||||
},
|
||||
}
|
||||
evidence = ROOT / "evidence" / "pipeline-faults.json"
|
||||
evidence.parent.mkdir(exist_ok=True)
|
||||
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
print(f"Evidence written to {evidence.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify explicit replay changes the deployed pipeline while preserving the original."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
MARK_FAILED_CODE = """
|
||||
import sys
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
with store.pool.connection() as conn:
|
||||
updated = conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', "
|
||||
"variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'",
|
||||
(sys.argv[1],),
|
||||
)
|
||||
if updated.rowcount != 1:
|
||||
raise RuntimeError("test attachment was not ready for failed replay injection")
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
AUDIT_CODE = """
|
||||
import json
|
||||
import sys
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
with store.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events "
|
||||
"WHERE attachment_id = %s ORDER BY id",
|
||||
(sys.argv[1],),
|
||||
).fetchall()
|
||||
print(json.dumps(rows, sort_keys=True))
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
|
||||
def docker_exec(container_id, *args):
|
||||
return subprocess.run(
|
||||
["docker", "exec", container_id, *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def replay(container_id, attachment_id, version):
|
||||
result = docker_exec(
|
||||
container_id,
|
||||
"python3",
|
||||
"-m",
|
||||
"src.replay_attachment",
|
||||
"--attachment-id",
|
||||
attachment_id,
|
||||
"--pipeline-version",
|
||||
str(version),
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify versioned attachment replay against the running Compose stack")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
ready_v1 = client.wait_ready(alice, attachment)
|
||||
v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail")
|
||||
v1_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch")
|
||||
|
||||
containers = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.split()
|
||||
check(bool(containers), "no running API container found")
|
||||
replay_v2 = replay(containers[0], attachment["id"], 2)
|
||||
check(replay_v2["status"] == "queued", "replay was not queued")
|
||||
check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2")
|
||||
check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id")
|
||||
|
||||
ready_v2 = client.wait_ready(alice, ready_v1)
|
||||
v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail")
|
||||
v2_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
original_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding")
|
||||
check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch")
|
||||
check(original_bytes == original, "pipeline replay changed the original file")
|
||||
|
||||
docker_exec(
|
||||
containers[0],
|
||||
"python3",
|
||||
"-c",
|
||||
MARK_FAILED_CODE,
|
||||
attachment["id"],
|
||||
)
|
||||
failed_replay = replay(containers[0], attachment["id"], 2)
|
||||
check(failed_replay["status"] == "queued", "failed attachment replay was not queued")
|
||||
check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version")
|
||||
check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id")
|
||||
ready_replayed = client.wait_ready(alice, ready_v2)
|
||||
audit = json.loads(
|
||||
docker_exec(
|
||||
containers[0],
|
||||
"python3",
|
||||
"-c",
|
||||
AUDIT_CODE,
|
||||
attachment["id"],
|
||||
).stdout
|
||||
)
|
||||
check(
|
||||
[(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit]
|
||||
== [(1, 2, "replay"), (2, 2, "replay")],
|
||||
"pipeline replay audit trail is incomplete",
|
||||
)
|
||||
check(ready_replayed["status"] == "ready", "failed attachment did not recover")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"previous_job_id": replay_v2["previous_job_id"],
|
||||
"job_id": replay_v2["job_id"],
|
||||
"pipeline_version": replay_v2["pipeline_version"],
|
||||
"failed_replay_job_id": failed_replay["job_id"],
|
||||
"audited_replays": len(audit),
|
||||
"v1_thumbnail_sha256": v1_thumbnail["sha256"],
|
||||
"v2_thumbnail_sha256": v2_thumbnail["sha256"],
|
||||
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic tests of the reconciliation decision rules without deleting real objects."""
|
||||
import os
|
||||
import hashlib
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret")
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from src.reconcile import reconcile
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
def __init__(self, items):
|
||||
self.items = items
|
||||
self.deleted = []
|
||||
|
||||
def objects(self):
|
||||
return iter(self.items)
|
||||
|
||||
def delete(self, key):
|
||||
self.deleted.append(key)
|
||||
self.items = [(name, date) for name, date in self.items if name != key]
|
||||
|
||||
def put(self, key, content, content_type):
|
||||
self.items.append((key, datetime.now(timezone.utc)))
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self):
|
||||
now = datetime.now(timezone.utc)
|
||||
self.metadata = {
|
||||
"original/present": {"id": "present-id", "object_key": "original/present"},
|
||||
"original/missing": {
|
||||
"id": "missing-id",
|
||||
"object_key": "original/missing",
|
||||
"size_bytes": 4,
|
||||
"sha256": hashlib.sha256(b"test").hexdigest(),
|
||||
"content_type": "image/png",
|
||||
},
|
||||
}
|
||||
self.objects = FakeObjects(
|
||||
[
|
||||
("original/present", now),
|
||||
("original/stale-orphan", now - timedelta(hours=2)),
|
||||
("original/new-orphan", now - timedelta(minutes=10)),
|
||||
]
|
||||
)
|
||||
self.missing_incidents = set()
|
||||
self.cleared_incidents = set()
|
||||
|
||||
def reconciliation_snapshot(self):
|
||||
return self.metadata
|
||||
|
||||
def record_missing_object(self, attachment_id):
|
||||
self.missing_incidents.add(attachment_id)
|
||||
|
||||
def clear_missing_object(self, attachment_id):
|
||||
self.missing_incidents.discard(attachment_id)
|
||||
self.cleared_incidents.add(attachment_id)
|
||||
|
||||
|
||||
def main():
|
||||
store = FakeStore()
|
||||
dry_run = reconcile(store, grace_seconds=3600, apply_changes=False)
|
||||
assert dry_run["stale_orphan_candidates"] == 1
|
||||
assert dry_run["missing_objects"] == 1
|
||||
assert dry_run["deleted_orphans"] == 0
|
||||
assert store.objects.deleted == [] and store.missing_incidents == set()
|
||||
|
||||
applied = reconcile(store, grace_seconds=3600, apply_changes=True)
|
||||
assert store.objects.deleted == ["original/stale-orphan"]
|
||||
assert "original/new-orphan" not in store.objects.deleted
|
||||
assert store.missing_incidents == {"missing-id"}
|
||||
assert store.cleared_incidents == {"present-id"}
|
||||
assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1
|
||||
|
||||
with tempfile.TemporaryDirectory() as backup:
|
||||
restore_dir = Path(backup)
|
||||
(restore_dir / "missing-id").write_bytes(b"test")
|
||||
preview = reconcile(store, 3600, False, restore_dir)
|
||||
assert preview["restorable_missing_objects"] == 1
|
||||
assert store.missing_incidents == {"missing-id"}
|
||||
restored = reconcile(store, 3600, True, restore_dir)
|
||||
assert restored["restored_missing_objects"] == 1
|
||||
assert store.missing_incidents == set()
|
||||
|
||||
repeated = reconcile(store, grace_seconds=3600, apply_changes=True)
|
||||
assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0
|
||||
assert store.objects.deleted == ["original/stale-orphan"]
|
||||
assert store.missing_incidents == set()
|
||||
print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
STORE_CODE = """
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
action = sys.argv[1]
|
||||
if action == "key":
|
||||
attachment_id = sys.argv[2]
|
||||
matches = [
|
||||
key for key, row in store.reconciliation_snapshot().items()
|
||||
if str(row["id"]) == attachment_id and row["reconcile_type"] == "original"
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("attachment metadata lookup failed")
|
||||
print(matches[0])
|
||||
elif action == "put":
|
||||
key, file_path = sys.argv[2:4]
|
||||
content = Path(file_path).read_bytes()
|
||||
store.objects.put(key, content, "image/png", len(content))
|
||||
elif action == "delete":
|
||||
store.objects.delete(sys.argv[2])
|
||||
elif action == "exists":
|
||||
print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no")
|
||||
else:
|
||||
raise RuntimeError("unsupported test operation")
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
PRIVATE_CHECK_CODE = """
|
||||
import socket
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
socket.getaddrinfo("s3-core", 8888)
|
||||
except socket.gaierror:
|
||||
print("PASS: SeaweedFS master/filer network is isolated")
|
||||
else:
|
||||
raise RuntimeError("API network can resolve the SeaweedFS core service")
|
||||
|
||||
for port in (8181, 9101):
|
||||
try:
|
||||
socket.create_connection(("s3", port), timeout=1).close()
|
||||
except OSError:
|
||||
continue
|
||||
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
|
||||
|
||||
try:
|
||||
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 403:
|
||||
print("PASS: unauthenticated bucket access denied")
|
||||
else:
|
||||
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
|
||||
else:
|
||||
raise RuntimeError("private bucket allowed an unauthenticated GET")
|
||||
"""
|
||||
|
||||
|
||||
def docker_exec(container_id, *args):
|
||||
return subprocess.run(
|
||||
["docker", "exec", container_id, *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def reconciliation(container_id, *args):
|
||||
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def object_exists(container_id, key):
|
||||
return docker_exec(
|
||||
container_id, "python3", "-c", STORE_CODE, "exists", key
|
||||
).stdout.strip() == "yes"
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
containers = result.stdout.split()
|
||||
check(len(containers) >= 2, "at least two running API replicas are required")
|
||||
container_id = containers[0]
|
||||
key = docker_exec(
|
||||
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
|
||||
).stdout.strip()
|
||||
orphan_key = "original/" + str(uuid.uuid4())
|
||||
|
||||
with tempfile.TemporaryDirectory() as backup:
|
||||
source = Path(backup) / "original.png"
|
||||
source.write_bytes(content)
|
||||
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
|
||||
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
|
||||
recovery_file = recovery_dir + "/" + attachment["id"]
|
||||
orphan_created = False
|
||||
orphan_removed = False
|
||||
missing_deleted = False
|
||||
missing_restored = False
|
||||
try:
|
||||
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
|
||||
orphan_created = True
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
|
||||
missing_deleted = True
|
||||
|
||||
preview = reconciliation(container_id)
|
||||
check(
|
||||
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
|
||||
"dry-run missed injected inconsistencies",
|
||||
)
|
||||
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
|
||||
immediate_apply = reconciliation(container_id, "--apply")
|
||||
check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period")
|
||||
|
||||
docker_exec(container_id, "mkdir", "-p", recovery_dir)
|
||||
subprocess.run(
|
||||
["docker", "cp", str(source), container_id + ":" + recovery_file],
|
||||
check=True,
|
||||
)
|
||||
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
|
||||
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
|
||||
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
||||
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
|
||||
missing_restored = True
|
||||
|
||||
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
||||
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
|
||||
check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period")
|
||||
print("Waiting for the 60-second orphan grace period.")
|
||||
time.sleep(61)
|
||||
|
||||
cleanup_preview = reconciliation(container_id)
|
||||
check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace")
|
||||
cleaned = reconciliation(container_id, "--apply")
|
||||
orphan_removed = not object_exists(container_id, orphan_key)
|
||||
check(orphan_removed, "injected stale orphan was not removed")
|
||||
final_scan = reconciliation(container_id, "--apply")
|
||||
check(
|
||||
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
|
||||
"repeat scan is not clean",
|
||||
)
|
||||
finally:
|
||||
if missing_deleted and not missing_restored:
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
|
||||
if orphan_created and not orphan_removed:
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
|
||||
|
||||
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
|
||||
check("PASS:" in private, "unauthenticated S3 access check failed")
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=bob,
|
||||
)["bytes"]
|
||||
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
|
||||
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"dataset_objects_scanned": final_scan["objects_seen"],
|
||||
"metadata_records_scanned": final_scan["metadata_seen"],
|
||||
"scan_duration_ms": final_scan["duration_ms"],
|
||||
"orphan_detected_and_removed": cleaned["deleted_orphans"],
|
||||
"missing_detected_and_restored": restored["restored_missing_objects"],
|
||||
"fresh_orphan_preserved_until_grace": True,
|
||||
"message_attachment_download_checksum": "unchanged",
|
||||
"private_bucket": "anonymous request denied",
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local transformation checks for the lab5 image worker."""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret")
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from src.worker import build_variants
|
||||
|
||||
|
||||
def main():
|
||||
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
attachment_id = "00000000-0000-4000-8000-000000000001"
|
||||
variants = build_variants(attachment_id, original, pipeline_version=1)
|
||||
by_name = {item["name"]: item for item in variants}
|
||||
assert set(by_name) == {"thumbnail", "crop", "optimized"}
|
||||
assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160)
|
||||
assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128)
|
||||
assert by_name["optimized"]["content_type"] == "image/webp"
|
||||
for item in variants:
|
||||
content = item["content"]
|
||||
assert len(content) == item["size_bytes"]
|
||||
assert hashlib.sha256(content).hexdigest() == item["sha256"]
|
||||
with Image.open(io.BytesIO(content)) as image:
|
||||
assert image.size == (item["width"], item["height"])
|
||||
newer = build_variants(attachment_id, original, pipeline_version=2)
|
||||
assert all(f"/v2/" in item["object_key"] for item in newer)
|
||||
assert all(f"/v1/" in item["object_key"] for item in variants)
|
||||
assert by_name["thumbnail"]["sha256"] != {
|
||||
item["name"]: item for item in newer
|
||||
}["thumbnail"]["sha256"]
|
||||
|
||||
odd_ratio = io.BytesIO()
|
||||
Image.new("RGB", (319, 200), "blue").save(odd_ratio, format="PNG")
|
||||
odd_variants = {
|
||||
item["name"]: item for item in build_variants(attachment_id, odd_ratio.getvalue())
|
||||
}
|
||||
assert (odd_variants["thumbnail"]["width"], odd_variants["thumbnail"]["height"]) == (256, 160)
|
||||
|
||||
small_image = io.BytesIO()
|
||||
Image.new("RGB", (100, 50), "red").save(small_image, format="PNG")
|
||||
small_variants = {
|
||||
item["name"]: item for item in build_variants(attachment_id, small_image.getvalue())
|
||||
}
|
||||
assert (small_variants["thumbnail"]["width"], small_variants["thumbnail"]["height"]) == (100, 50)
|
||||
assert (small_variants["optimized"]["width"], small_variants["optimized"]["height"]) == (100, 50)
|
||||
print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify queued uploads and message links survive an API/worker restart."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify queued attachment recovery after API and worker restart")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
compose("stop", "worker")
|
||||
worker_stopped = True
|
||||
try:
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload was not accepted while worker was stopped")
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message")
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
409,
|
||||
actor=bob,
|
||||
)
|
||||
compose("restart", "api")
|
||||
compose("start", "worker")
|
||||
worker_stopped = False
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
original_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=bob,
|
||||
)["bytes"]
|
||||
check(original_bytes == original, "recovered worker changed original bytes")
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
recovered_message = next(item for item in page["items"] if item["id"] == message["id"])
|
||||
check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"initial_status": attachment["status"],
|
||||
"final_status": ready["status"],
|
||||
"message_attachment_link_survived": True,
|
||||
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if worker_stopped:
|
||||
compose("start", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
|
||||
|
||||
|
||||
def usage_bytes(line):
|
||||
match = USAGE_RE.search(line)
|
||||
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
|
||||
|
||||
|
||||
def api_containers():
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
ids = result.stdout.split()
|
||||
check(len(ids) >= 2, "at least two running API containers are required")
|
||||
return ids
|
||||
|
||||
|
||||
def sample_once(ids):
|
||||
output = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.splitlines()
|
||||
values = [usage_bytes(line) for line in output]
|
||||
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
|
||||
return max(values)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Bounded-memory ten-client upload experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
width = height = 1400
|
||||
content = BytesIO()
|
||||
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
|
||||
content, format="PNG", compress_level=0
|
||||
)
|
||||
payload = content.getvalue()
|
||||
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
|
||||
ids = api_containers()
|
||||
baseline = sample_once(ids)
|
||||
samples = []
|
||||
stats_process = subprocess.Popen(
|
||||
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
|
||||
def sampler():
|
||||
if stats_process.stdout is None:
|
||||
return
|
||||
for line in stats_process.stdout:
|
||||
value = usage_bytes(line)
|
||||
if value:
|
||||
samples.append(value)
|
||||
|
||||
sampling_thread = threading.Thread(target=sampler, daemon=True)
|
||||
sampling_thread.start()
|
||||
count = 10
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def upload(index):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=15)
|
||||
return worker.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
|
||||
202,
|
||||
actor=alice,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(upload, range(count)))
|
||||
finally:
|
||||
stopped_by_test = False
|
||||
if stats_process.poll() is None:
|
||||
stats_process.kill()
|
||||
stopped_by_test = True
|
||||
try:
|
||||
stats_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
stats_process.kill()
|
||||
stats_process.wait(timeout=5)
|
||||
sampling_thread.join(timeout=5)
|
||||
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
|
||||
check(
|
||||
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
|
||||
"docker stats sampling process failed (exit "
|
||||
+ str(stats_process.returncode)
|
||||
+ "): "
|
||||
+ stats_error.strip()[:300],
|
||||
)
|
||||
check(bool(samples), "docker stats produced no memory samples")
|
||||
peak = max([baseline] + samples)
|
||||
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"uploads": count,
|
||||
"concurrency": count,
|
||||
"bytes_per_upload": len(payload),
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"api_containers": len(ids),
|
||||
"sample_count": len(samples) // len(ids),
|
||||
"baseline_max_container_bytes": baseline,
|
||||
"peak_max_container_bytes": peak,
|
||||
"peak_delta_bytes": max(0, peak - baseline),
|
||||
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
|
||||
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user