ЛР 5 — Группа БВТ2304 — Аккуратов — на 5 #17

Open
kaktuz2000 wants to merge 2 commits from kaktuz2000/mtusi2026lab5:solution/bvt2304-akkuratov into main
48 changed files with 5016 additions and 53 deletions
+8 -19
View File
@@ -1,25 +1,14 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
API_CPU_LIMIT=1.0
API_MEMORY_LIMIT=512M
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
MAX_CONCURRENT_UPLOADS=10
WORKER_CONCURRENCY=1
WORKER_REPLICAS=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
RECONCILE_INTERVAL_SECONDS=900
RECONCILE_GRACE_SECONDS=3600
+27 -11
View File
@@ -4,15 +4,31 @@
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: Перенесены собственные сервер, PostgreSQL, S3, Compose-конфигурация и базовые тесты из предыдущей лабораторной. Сохранены переходы API по контракту lab5: загрузка вложения возвращает 202 и обрабатывается асинхронно. Добавлены RabbitMQ publisher/worker, transactional outbox, retry/recovery, versioned pipeline v1/v2, fencing устаревших задач, безопасный replay failed-вложений с audit trail и fault-injection тесты. README, COURSE, contracts и публичные tests задания не заменялись.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Команды запуска из чистого clone:
```sh
git clone https://gitea.privcloud.tech/kaktuz2000/mtusi2026lab5.git
cd mtusi2026lab5
git switch solution/bvt2304-akkuratov
cp .env.example .env
scripts/bootstrap-secrets.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
until curl --fail --silent http://localhost:${APP_PORT:-8080}/health/ready >/dev/null; do sleep 2; done
make check
make test BASE_URL=http://localhost:${APP_PORT:-8080}
```
Результаты проверок и ссылка на REPORT.md: [REPORT.md](../../REPORT.md). `make check` — PASS; публичный smoke `make test BASE_URL=http://localhost:18080` — PASS (`PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.`). Также PASS: 8 сценариев fault injection, outbox/worker recovery, replay v1→v2 и failed replay с двумя audit-событиями, повреждённые вложения и S3 ACL, persistence, API failover. Фактические результаты: [evidence/lab5-integration.json](../../evidence/lab5-integration.json), [evidence/pipeline-faults.json](../../evidence/pipeline-faults.json), [evidence/load.json](../../evidence/load.json).
Ограничения / известные ошибки: Замер нагрузки — 30-секундный closed-loop тест на одной машине, без отдельного прогрева; он не является оценкой open-loop или production capacity. Другие известные ошибки по выполненным проверкам не обнаружены.
Источники и использование ИИ: Перенесён собственный код из mtusi2026lab4; требования и публичные проверки взяты из README.md, COURSE.md, CONTRIBUTING.md и tests репозитория mtusi2026lab5. Использован ИИ-ассистент для переноса, адаптации реализации, написания дополнительных тестов и отчёта. Внешние фрагменты кода не добавлялись.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+22
View File
@@ -0,0 +1,22 @@
FROM python:3.12.7-slim
WORKDIR /app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY . /app
RUN chown -R app:app /app
USER app
EXPOSE 8080
CMD ["python3", "src/server.py"]
+33 -2
View File
@@ -4,10 +4,11 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-attachments test-idempotency-race test-reconcile test-worker test-pipeline-faults test-pipeline-replay test-outbox-recovery test-worker-recovery \
test-reconcile-integration test-upload-concurrency load test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check/validate: offline contract checks; test: public smoke; student targets: focused attachment/pipeline checks"
check:
$(PYTHON) scripts/check_contract.py
@@ -18,6 +19,36 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency-race:
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile:
$(PYTHON) student-tests/test_reconcile.py
test-worker:
$(PYTHON) student-tests/test_worker.py
test-pipeline-faults:
$(PYTHON) student-tests/test_pipeline_faults.py
test-pipeline-replay:
$(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-outbox-recovery:
$(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-worker-recovery:
$(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile-integration:
$(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-upload-concurrency:
$(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
+70 -21
View File
@@ -1,37 +1,86 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Группа и фамилия: BVT2304, Akkuratov.
- Целевая оценка: 5.
- Выбранный трек: 2 — доказанная идемпотентность pipeline.
- Commit/PR: не создан.
- Использованные библиотеки: psycopg, redis, boto3, Pillow, pika; версии закреплены в `requirements.txt`.
- В работе использовался ИИ-ассистент для переноса, адаптации и проверки кода.
- Проверенный стенд: Python 3.12.7; PostgreSQL 16.4; RabbitMQ 3.13.7; SeaweedFS 4.51; HAProxy 3.0.7; Docker Desktop 28.0.4; 2 API-реплики. Хост: Apple M2 Pro (10 ядер, 16 GiB RAM), macOS 27.0.1.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Python 3.9+ и Docker Compose v2. Полные команды есть в [code.md](./code.md). Основной запуск:
```sh
cp .env.example .env
scripts/bootstrap-secrets.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
until curl --fail --silent http://localhost:8080/health/ready >/dev/null; do sleep 2; done
make test BASE_URL=http://localhost:8080
```
Секреты создаются локально и не публикуются. Миграции выполняются сервисом `migrate`; остановка без удаления данных — `docker compose down`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
PostgreSQL — источник истины для предметных данных, состояния вложения и transactional outbox. Подтверждённая API-загрузка сохраняет исходник в S3 staging и метаданные с outbox-записью до ответа `202 queued`. Publisher передаёт durable RabbitMQ job с ID и служебными метаданными, без bytes и credentials. Worker декодирует изображение в отдельном процессе с ограничением времени, записывает оригинал и `thumbnail`/`crop`/`optimized`, затем фиксирует `ready`. Повторы ограничены, используют backoff с jitter; lease и publisher восстанавливают просроченную работу.
## Доказательства критериев
Обработка ограничена `JOB_TIMEOUT_SECONDS`; дочерний процесс принудительно завершается при превышении. S3-клиент задаёт connect/read timeout и число retries, lease включает отдельный сетевой бюджет. Worker имеет memory limit 1 GiB и concurrency по умолчанию 1; API ограничивает файл 10 MiB и изображение 40 млн пикселей.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Версии pipeline задают отдельные параметры и S3-prefix для вариантов: v1 использует JPEG quality 85/WebP 80, v2 — JPEG 80/WebP 75. Геометрия остаётся одинаковой. Операторский replay создаёт новый `job_id`, сбрасывает попытку, публикует новую outbox-запись и добавляет audit event. SQL completion/failure ограничены текущими job ID, pipeline version и attempt; поздняя задача не может изменить текущее состояние. Reconciler периодически сверяет PostgreSQL и S3. Filer доступен только из внутренней Compose-сети.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Трек 2: гипотеза и методика
## Эксперименты и отказы
**Гипотеза:** повторная доставка может повторить физические записи, но уникальные versioned object keys и fencing по `(attachment_id, job_id, pipeline_version, attempt)` не допускают потери принятой работы или публикации устаревшего результата.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Методика: `make test-pipeline-faults` запускает fault injection на штатном consumer/publisher-коде и контролируемых брокере, объектном хранилище и state-store. Используется публичный синтетический fixture `sample.png`. Падения вводятся до первой побочной записи, после записи оригинала и после фиксации результата до ack; дополнительно проверяются повторы, reorder, смена версии и неопределённый publisher confirm. Результат каждого запуска записывается в [evidence/pipeline-faults.json](./evidence/pipeline-faults.json).
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Сценарий | Наблюдение |
| --- | --- |
| Дубль после commit до ack | 2 доставки, 4 записи объектов для одного набора; повторная доставка — 0 дополнительных записей |
| Crash сразу после claim | lease recovery создал attempt 2; повтор завершился `ready` |
| Crash сразу после записи оригинала | повторная запись детерминированна; attempt 2 завершился `ready` |
| Reorder attempt 2 затем attempt 1 | устаревшая попытка: 0 дополнительных записей |
| v1 superseded версией v2 во время обработки | metadata ссылается только на `v2` variants |
| Publisher crash до publish | pending outbox пережил перезапуск, задание опубликовано после восстановления |
| Confirm потерян после broker accept | два сообщения имеют идентичное тело и стабильный message ID; worker no-op для дубля |
| Ограничение времени transform | зависший дочерний процесс завершён за 1.008 секунды при бюджете 1 секунда |
## Ограничения и следующие шаги
Compose integration PASS: `make test-outbox-recovery` подтвердил переход queued→ready после принятого upload при остановленном publisher; `make test-pipeline-replay` выполнил v1→v2, повторно обработал failed-вложение на той же версии и проверил два audit-события. Исходный checksum сохранился. `make test-worker-recovery` подтвердил queued-вложение после перезапуска API/worker. Фактические результаты проверок стенда сохранены в [evidence/lab5-integration.json](./evidence/lab5-integration.json).
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## Нагрузка
Повторяемый замер: `.venv/bin/python tests/load.py --base-url http://localhost:18080 --duration 30 --concurrency 4 --output evidence/load.json`. После готовности API и подготовки тестовых клиентов выполнен 30-секундный closed-loop POST-тест на синтетических сообщениях: 18,768 успешных запросов, 0 ошибок, 625.458 успешных RPS; latency успешных запросов — p50 5.816 ms, p95 8.820 ms, p99 19.155 ms. Дополнительно `docker stats` измерял CPU и память контейнеров Compose: сумма CPU peak/mean — 322.07%/232.49%, сумма памяти peak/mean — 890,263,039/814,641,644 байт, 11 выборок (примерно раз в 2.73 секунды). API: две реплики, лимит каждой 1 CPU/512 MiB; worker memory limit 1 GiB. Отдельная предварительная фаза прогрева не проводилась. Это closed-loop результат на указанном хосте; его нельзя напрямую трактовать как open-loop capacity или гарантированный production throughput. Полные данные: [evidence/load.json](./evidence/load.json).
## Инварианты и доставка
- `202` означает, что исходник и намерение обработки устойчиво сохранены; publisher можно запускать позже.
- Job повторяется at-least-once. После потери confirm одно сообщение может быть опубликовано дважды; тест намеренно наблюдает две доставки.
- Результаты представлены детерминированными S3 keys, различными для pipeline versions; metadata переключается только условным SQL update.
- Ack отправляется только после устойчивого результата или после записи долговременного retry/outbox намерения. Дубль уже завершённой задачи — no-op.
- Это не exactly-once delivery: между broker accept и фиксацией publisher confirm возможна повторная публикация, а между SQL commit результата и consumer ack возможна повторная доставка. Гарантируется fencing/idempotency эффекта, не единственная доставка или единственное выполнение.
## Проверки
| Проверка | Команда | Результат |
| --- | --- | --- |
| Контракт skeleton | `make check` | PASS |
| OpenAPI/JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS |
| Варианты v1/v2 | `make test-worker PYTHON=.venv/bin/python` | PASS |
| Reconciler unit tests | `make test-reconcile PYTHON=.venv/bin/python` | PASS |
| Track 2 injection | `make test-pipeline-faults PYTHON=.venv/bin/python` | PASS; 8 сценариев |
| Python syntax/whitespace | `compileall`, `git diff --check` | PASS |
| Compose parse | `docker compose --env-file .env.example config --quiet` | PASS |
| Public HTTP smoke | `make test BASE_URL=http://localhost:18080` | PASS |
| Ошибочные загрузки/ACL | `make test-attachments BASE_URL=http://localhost:18080` | PASS |
| Concurrent idempotency | `make test-idempotency-race BASE_URL=http://localhost:18080` | PASS; 12 запросов, 2 API, одно сообщение |
| Outbox после publisher stop | `make test-outbox-recovery BASE_URL=http://localhost:18080` | PASS; queued→ready без ручной публикации |
| Очередь после API/worker restart | `make test-worker-recovery BASE_URL=http://localhost:18080` | PASS; ссылка сообщения и оригинал сохранены |
| Реальный replay v1→v2 | `make test-pipeline-replay BASE_URL=http://localhost:18080` | PASS; разные thumbnail SHA-256, оригинал совпал |
| Повтор failed-вложения и audit | `make test-pipeline-replay BASE_URL=http://localhost:18080` | PASS; failed→queued→ready, два audit-события |
| Reconciliation/S3 ACL | `make test-reconcile-integration BASE_URL=http://localhost:18080` | PASS; orphan cleanup, exact restore, анонимный GET запрещён |
| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080` | PASS; данные и сессия пережили restart |
| API failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080` | PASS; 131 запись, восстановление за 0.681 s |
| Параллельная загрузка | `make test-upload-concurrency PYTHON=.venv/bin/python BASE_URL=http://localhost:18080` | PASS; 10 × 5,883,411 байт |
+103
View File
@@ -0,0 +1,103 @@
# Запуск и проверка лабораторной 5
## Подготовка стенда
Нужны Docker Compose v2 и доступный Docker daemon. Все команды выполняются из корня репозитория.
```sh
cp .env.example .env
scripts/bootstrap-secrets.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
```
Секреты создаются локально в `secrets/` с правами владельца и не выводятся в терминал. Не добавляйте `.env` или `secrets/` в Git. Compose поднимает PostgreSQL, RabbitMQ, SeaweedFS, миграции, API, publisher, worker, периодический reconciler и HAProxy.
Дождитесь готовности API и проверьте состояние сервисов:
```sh
until curl --fail --silent http://localhost:8080/health/ready >/dev/null; do sleep 2; done
docker compose ps
curl --fail --silent http://localhost:8080/health/ready
```
Если порт `8080` занят, поменяйте `APP_PORT` в локальном `.env` (например, на `18080`) и используйте этот порт во всех командах ниже.
## Проверки кода
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make check PYTHON=.venv/bin/python
make validate PYTHON=.venv/bin/python
make test-worker PYTHON=.venv/bin/python
make test-reconcile PYTHON=.venv/bin/python
make test-pipeline-faults PYTHON=.venv/bin/python
```
`test-pipeline-faults` исполняет fault injection для дубля, reorder, падения после claim и после записи оригинала, потери ack, гонки старой/новой версии, остановки publisher до publish, потери publisher confirm и hard timeout обработки. Санитизированный JSON создаётся в `evidence/pipeline-faults.json`.
## API и интеграционные проверки
Стенд должен быть запущен. Тесты создают синтетические данные и не удаляют их; используйте учебный стенд.
```sh
make test BASE_URL=http://localhost:8080
make test-attachments BASE_URL=http://localhost:8080
make test-idempotency-race BASE_URL=http://localhost:8080
make test-reconcile-integration BASE_URL=http://localhost:8080
make test-upload-concurrency BASE_URL=http://localhost:8080
make load BASE_URL=http://localhost:8080
# Exact 30-second/4-client benchmark, saving sanitized metrics:
.venv/bin/python tests/load.py --base-url http://localhost:8080 \
--duration 30 --concurrency 4 --output evidence/load.json
```
Для проверок уровня 4/5:
```sh
# Подтверждённая API загрузка при остановленном publisher должна дойти до ready после его старта.
make test-outbox-recovery BASE_URL=http://localhost:8080
# Upload/message при остановленном worker; restart API и worker, затем ready и download.
make test-worker-recovery BASE_URL=http://localhost:8080
# Готовое вложение переобрабатывается pipeline v2 с новой версией job.
make test-pipeline-replay BASE_URL=http://localhost:8080
```
Pipeline v1 использует JPEG quality 85 и WebP quality 80; v2 — JPEG quality 80 и WebP quality 75. Геометрия вариантов остаётся фиксированной контрактом. При replay `job_id` меняется, попытки начинаются заново, версии вариантов получают разные S3-prefix (`variants/<attachment-id>/vN/`), а событие записывается в PostgreSQL. В рабочем контейнере явный replay выполняется так:
```sh
docker compose exec --index 1 -T api python3 -m src.replay_attachment \
--attachment-id "$ATTACHMENT_ID" --pipeline-version 2
```
Перед запуском задайте `ATTACHMENT_ID` равным UUID существующего вложения из ответа API.
Для повторной попытки failed-вложения укажите его текущую версию; переход на уже опубликованную версию разрешён только для failed, повышение версии supersede-ит старую активную задачу.
## Отказ и сохранность
```sh
chmod +x scripts/restart.sh scripts/stop-one.sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
docker compose up -d --scale api=2 api
```
Сценарий failover оставляет выбранный API-контейнер остановленным; последняя команда возвращает две реплики. Restart hook перезапускает PostgreSQL, SeaweedFS, RabbitMQ, API, publisher и worker без удаления volumes.
## Диагностика и остановка
```sh
docker compose logs -f api publisher worker rabbitmq reconciler
docker compose down
```
`docker compose down` сохраняет named volumes. Для намеренного удаления тестовой БД, очереди и S3-файлов:
```sh
docker compose down --volumes
```
Эту команду выполняйте только если данные стенда больше не нужны.
+352
View File
@@ -0,0 +1,352 @@
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 20
start_period: 5s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
rabbitmq:
image: rabbitmq:3.13.7-management-alpine
secrets:
- source: rabbitmq_config
target: /etc/rabbitmq/conf.d/10-defaults.conf
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
restart: unless-stopped
s3-core:
image: chrislusf/seaweedfs:4.51
entrypoint: ["weed"]
command:
- server
- -dir=/data
- -filer=true
- -s3=false
volumes:
- seaweed_data:/data
healthcheck:
test:
- CMD-SHELL
- "curl --fail --silent http://127.0.0.1:8888/ >/dev/null"
interval: 3s
timeout: 2s
retries: 30
start_period: 10s
networks:
- seaweed_internal
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
s3:
image: chrislusf/seaweedfs:4.51
entrypoint: ["weed"]
command:
- s3
- -filer=s3-core:8888
- -config=/run/secrets/s3_config
- -port=8333
- -port.https=0
- -port.grpc=0
- -port.iceberg=0
- -port.lance=0
- -concurrentFileUploadLimit=10
- -concurrentUploadLimitMB=100
- -readerCacheSizeMB=64
- -autoCreateBucket=false
- -ip.bind=0.0.0.0
secrets:
- s3_config
depends_on:
s3-core:
condition: service_healthy
networks:
- default
- seaweed_internal
healthcheck:
test:
- CMD-SHELL
- "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]"
interval: 3s
timeout: 2s
retries: 30
start_period: 10s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
s3-bootstrap:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/bootstrap_s3.py"]
environment:
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger-attachments
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger-attachments
DB_POOL_MAX_SIZE: "2"
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
restart: "no"
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10}
UPLOAD_TIMEOUT_SECONDS: "60"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MAX_SIZE: "8"
DB_POOL_TIMEOUT_SECONDS: "3"
DB_STATEMENT_TIMEOUT_MS: "5000"
DB_LOCK_TIMEOUT_MS: "2000"
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
healthcheck:
test:
- CMD
- python3
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"
interval: 3s
timeout: 3s
retries: 5
start_period: 10s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPU_LIMIT:-1.0}"
memory: "${API_MEMORY_LIMIT:-512M}"
stop_grace_period: 20s
restart: unless-stopped
publisher:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "-m", "src.publisher"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
DB_POOL_MAX_SIZE: "2"
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
secrets:
- database_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
worker:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "-m", "src.worker"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
MAX_IMAGE_PIXELS: "40000000"
DB_POOL_MAX_SIZE: "2"
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
secrets:
- database_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
reconciler:
build:
context: .
dockerfile: Dockerfile
command: ["sh", "scripts/reconcile-loop.sh"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
DB_POOL_MAX_SIZE: "2"
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900}
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600}
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
load_balancer:
image: haproxy:3.0.7-alpine
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
restart: unless-stopped
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
s3_config:
file: ./secrets/s3.json
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
rabbitmq_password:
file: ./secrets/rabbitmq_password
rabbitmq_config:
file: ./secrets/rabbitmq.conf
broker_url:
file: ./secrets/broker_url
volumes:
postgres_data:
seaweed_data:
rabbitmq_data:
networks:
seaweed_internal:
internal: true
+120
View File
@@ -0,0 +1,120 @@
{
"date": "2026-11-04",
"compose": {
"api_replicas": 2,
"host_port": 18080,
"services_healthy": true,
"readiness": "200"
},
"fixture": {
"file": "tests/fixtures/sample.png",
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"size_bytes": 111817
},
"checks": [
{
"name": "public_smoke",
"result": "PASS",
"command": "make test BASE_URL=http://localhost:18080"
},
{
"name": "attachment_negative_cases",
"result": "PASS",
"command": "make test-attachments BASE_URL=http://localhost:18080",
"cases": [
"damaged PNG",
"spoofed MIME",
"oversized upload",
"pixel limit",
"unsafe filename",
"unauthorized chat",
"cross-chat attachment"
]
},
{
"name": "idempotency_race",
"result": "PASS",
"replicas": 2,
"concurrent_same_key_requests": 12,
"messages_created": 1
},
{
"name": "outbox_recovery",
"result": "PASS",
"status_before_publisher_restart": "queued",
"status_after_restart": "ready",
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"
},
{
"name": "worker_restart_recovery",
"result": "PASS",
"sequence": [
"stop worker",
"upload returns 202 queued",
"link attachment to message",
"restart API",
"start worker",
"attachment reaches ready"
],
"message_attachment_link_survived": true
},
{
"name": "pipeline_version_replay",
"result": "PASS",
"version_transition": "1 -> 2",
"job_id_changed": true,
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"v1_thumbnail_sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"v2_thumbnail_sha256": "7904b78c14d936a9e5b4d024bd7bf9ffe4de546c484b356177056b8e949e763f"
},
{
"name": "failed_pipeline_replay",
"result": "PASS",
"previous_job_id": "2628b822-ae4c-4784-b3bf-03f695176984",
"failed_replay_job_id": "797cc24f-0c3a-49fa-9f25-febd21506c6c",
"pipeline_version": 2,
"audited_replays": 2,
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"
},
{
"name": "reconciliation",
"result": "PASS",
"dataset_objects_scanned": 36,
"metadata_records_scanned": 9,
"scan_duration_ms": 20.873,
"orphan_detected_and_removed": 1,
"missing_object_detected_and_restored": 1,
"fresh_orphan_preserved_until_grace": true,
"anonymous_s3_access": "denied"
},
{
"name": "persistence_restart",
"result": "PASS",
"confirmed_data_preserved": true,
"existing_session_preserved": true
},
{
"name": "api_failover",
"result": "PASS",
"instances_before": 2,
"successful_writes": 131,
"failed_attempts": 1,
"first_recovery_seconds": 0.681,
"confirmed_data_preserved": true
},
{
"name": "concurrent_upload_memory",
"result": "PASS",
"api_replicas": 2,
"uploads": 10,
"concurrency": 10,
"bytes_per_upload": 5883411,
"duration_seconds": 2.525,
"baseline_max_container_bytes": 96101990,
"peak_max_container_bytes": 96101990,
"peak_delta_bytes": 0,
"sampling_method": "continuous docker stats stream"
}
],
"track_2_fault_injection": "pipeline-faults.json"
}
+43
View File
@@ -0,0 +1,43 @@
{
"lab": 5,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.007,
"concurrency": 4,
"successful_requests": 18768,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 625.458,
"successful_latency_ms": {
"p50": 5.816,
"p95": 8.82,
"p99": 19.155
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.",
"environment": {
"host": "Apple M2 Pro",
"host_cpu_cores": 10,
"host_ram_bytes": 17179869184,
"macos_version": "27.0.1",
"docker_desktop_version": "28.0.4",
"api_replicas": 2,
"api_cpu_limit_per_container": 1.0,
"api_memory_limit_per_container": "512M",
"worker_memory_limit": "1G"
},
"warmup": "No separate warm-up interval; run started after readiness and test-client bootstrap.",
"dataset": "18,768 successful synthetic POST message writes; concurrency 4.",
"resource_sampling": {
"sample_interval_seconds_approx": 2.73,
"samples": 11,
"cpu_percent_sum_all_compose_containers": {
"peak": 322.07,
"mean": 232.49
},
"memory_bytes_sum_all_compose_containers": {
"peak": 890263039,
"mean": 814641644
},
"source": "docker stats --no-stream; aggregate across running Compose containers"
}
}
+58
View File
@@ -0,0 +1,58 @@
{
"fixture": "tests/fixtures/sample.png",
"invariants": {
"ambiguous_confirm_republishes_stable_job": true,
"confirmed_work_eventually_ready": true,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
"duplicate_effects_create_no_second_variant_set": true,
"stale_attempt_writes_no_effects": true,
"stale_pipeline_cannot_publish_current_metadata": true
},
"result": "PASS",
"scenarios": [
{
"deliveries": 2,
"duplicate_delivery_writes": 0,
"effective_object_writes": 4,
"scenario": "duplicate_delivery_after_commit_before_ack"
},
{
"attempts": 2,
"scenario": "crash_after_claim",
"terminal_status": "ready"
},
{
"attempts": 2,
"scenario": "crash_after_original_put",
"terminal_status": "ready"
},
{
"current_attempt": 2,
"scenario": "out_of_order_attempt_delivery",
"stale_attempt_extra_writes": 0
},
{
"current_pipeline_version": 2,
"published_variant_versions": [
"v2"
],
"scenario": "stale_pipeline_after_newer_version"
},
{
"outbox_recovered_after_restart": true,
"published_after_restart": 1,
"scenario": "publisher_crash_before_publish"
},
{
"deliveries": 2,
"same_message_id": true,
"scenario": "publish_confirm_lost_after_broker_accept"
},
{
"configured_timeout_seconds": 1,
"observed_seconds": 1.008,
"scenario": "transform_timeout_terminated"
}
],
"track": "2"
}
+35
View File
@@ -0,0 +1,35 @@
global
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
option http-keep-alive
timeout connect 2s
timeout client 15s
timeout server 15s
timeout http-request 5s
timeout check 1s
retries 2
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold nx 2s
frontend messenger
bind :8080
default_backend api_replicas
backend api_replicas
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
option redispatch
default-server inter 1s fall 2 rise 2
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
+43
View File
@@ -0,0 +1,43 @@
CREATE TABLE users (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
username varchar(32) NOT NULL UNIQUE,
display_name varchar(100) NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
title varchar(100) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method varchar(8) NOT NULL,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
idempotency_key varchar(128) NOT NULL,
request_hash bytea NOT NULL,
response jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
+59
View File
@@ -0,0 +1,59 @@
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank;
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
filename varchar(255) NOT NULL,
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
staging_key text NOT NULL UNIQUE,
job_id uuid NOT NULL UNIQUE,
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
lease_until timestamptz,
status varchar(16) NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'),
error_code varchar(64),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, ordinal)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
CREATE TABLE attachment_outbox (
id uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
attempt integer NOT NULL CHECK (attempt >= 1),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
UNIQUE (job_id, attempt)
);
CREATE INDEX attachment_outbox_pending_idx
ON attachment_outbox (available_at, created_at)
WHERE published_at IS NULL;
CREATE TABLE reconciliation_incidents (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')),
first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, incident_type)
);
+16
View File
@@ -0,0 +1,16 @@
ALTER TABLE attachment_outbox
DROP CONSTRAINT attachment_outbox_job_id_fkey;
CREATE TABLE attachment_pipeline_events (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
prior_job_id uuid NOT NULL,
job_id uuid NOT NULL,
prior_version integer NOT NULL CHECK (prior_version >= 1),
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
action varchar(16) NOT NULL CHECK (action = 'replay'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachment_pipeline_events_attachment_created_idx
ON attachment_pipeline_events (attachment_id, created_at);
+6
View File
@@ -0,0 +1,6 @@
psycopg[binary]==3.2.10
psycopg-pool==3.3.3
redis==5.2.1
boto3==1.35.99
Pillow==11.0.0
pika==1.3.2
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
import json
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
def main():
SECRET_DIR.mkdir(mode=0o700, exist_ok=True)
names = (
"postgres_password",
"database_url",
"s3_access_key_id",
"s3_secret_access_key",
"s3_admin_access_key_id",
"s3_admin_secret_access_key",
"rabbitmq_password",
"rabbitmq.conf",
"broker_url",
"s3.json",
)
existing = [name for name in names if (SECRET_DIR / name).exists()]
if existing:
raise SystemExit(
"Secret files already exist; refusing to rotate credentials without a migration: "
+ ", ".join(existing)
)
db_password = secrets.token_hex(24)
broker_password = secrets.token_urlsafe(32)
values = {
"postgres_password": db_password,
"database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger",
"s3_access_key_id": "mngr" + secrets.token_hex(8),
"s3_secret_access_key": secrets.token_urlsafe(40),
"s3_admin_access_key_id": "admin" + secrets.token_hex(7),
"s3_admin_secret_access_key": secrets.token_urlsafe(40),
"rabbitmq_password": broker_password,
"broker_url": f"amqp://messenger:{broker_password}@rabbitmq:5672/%2f",
}
for name, value in values.items():
path = SECRET_DIR / name
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(value + "\n")
descriptor = os.open(
SECRET_DIR / "rabbitmq.conf",
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o600,
)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write("default_user = messenger\n")
output.write(f"default_pass = {broker_password}\n")
s3_config = {
"identities": [
{
"name": "bootstrap-admin",
"credentials": [
{
"accessKey": values["s3_admin_access_key_id"],
"secretKey": values["s3_admin_secret_access_key"],
}
],
"actions": ["Admin"],
},
{
"name": "messenger-api",
"credentials": [
{
"accessKey": values["s3_access_key_id"],
"secretKey": values["s3_secret_access_key"],
}
],
"actions": [
"Read:messenger-attachments",
"Write:messenger-attachments",
"List:messenger-attachments",
],
},
]
}
descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(s3_config, output, separators=(",", ":"))
output.write("\n")
print("Local secrets generated with owner-only permissions; values were not printed.")
if __name__ == "__main__":
main()
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 "$(dirname "$0")/bootstrap-secrets.py"
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
interval=${RECONCILE_INTERVAL_SECONDS:-900}
grace=${RECONCILE_GRACE_SECONDS:-3600}
case "$interval" in
''|*[!0-9]*)
echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2
exit 2
;;
esac
case "$grace" in
''|*[!0-9]*)
echo "RECONCILE_GRACE_SECONDS must be an integer" >&2
exit 2
;;
esac
if [ "$interval" -lt 60 ]; then
echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2
exit 2
fi
if [ "$grace" -lt 60 ]; then
echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2
exit 2
fi
while :
do
python3 src/reconcile.py --grace-seconds "$grace" --apply
sleep "$interval"
done
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose restart postgres s3-core rabbitmq
docker compose up -d --wait postgres s3-core rabbitmq
docker compose restart s3
docker compose up -d --wait s3
attempt=0
until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 4 ]; then
echo "S3 object reads did not recover after restart" >&2
exit 1
fi
sleep 2
done
docker compose restart api
docker compose restart publisher worker
docker compose up -d --wait api publisher worker
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
found=
for container_id in $(docker compose ps --quiet --status running api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
found=$container_id
break
fi
done
if [ -z "$found" ]; then
printf '%s\n' 'No running API container matched the requested instance alias.' >&2
exit 1
fi
docker stop --time 5 "$found" >/dev/null
+1
View File
@@ -0,0 +1 @@
"""Messenger API application package."""
+39
View File
@@ -0,0 +1,39 @@
import logging
import os
import signal
import threading
from functools import partial
from http.server import ThreadingHTTPServer
from .config import APP_HOST, APP_PORT
from .handlers import MessengerHandler
from .store import Store
LOGGER = logging.getLogger("messenger")
def run_server():
store = Store()
store.open()
try:
handler = partial(MessengerHandler, store=store)
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
def handle_shutdown(signum, frame):
LOGGER.info("received signal %s, shutting down gracefully", signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, handle_shutdown)
signal.signal(signal.SIGINT, handle_shutdown)
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
try:
server.serve_forever()
finally:
server.server_close()
finally:
store.close()
def main():
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
run_server()
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
import logging
import os
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
LOGGER = logging.getLogger("messenger.s3.bootstrap")
def secret(name: str) -> str:
direct = os.getenv(name)
if direct:
return direct
file_path = os.getenv(name + "_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError(f"Set {name} or {name}_FILE")
def main():
client = boto3.client(
"s3",
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
region_name="us-east-1",
aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"),
aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"),
config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}),
)
bucket = os.getenv("S3_BUCKET", "messenger-attachments")
try:
client.head_bucket(Bucket=bucket)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status != 404:
raise
client.create_bucket(Bucket=bucket)
client.put_bucket_acl(Bucket=bucket, ACL="private")
LOGGER.info("private S3 bucket bootstrap completed")
if __name__ == "__main__":
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
main()
+61
View File
@@ -0,0 +1,61 @@
import os
import re
import socket
import uuid
from pathlib import Path
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10"))
UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60"))
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
def secret_value(name: str, default: str = "") -> str:
value = os.getenv(name)
if value:
return value
file_path = os.getenv(name + "_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
if default:
return default
raise RuntimeError(f"Set {name} or {name}_FILE")
S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY")
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
if SESSION_TTL_SECONDS < 1:
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
if IDEMPOTENCY_TTL_SECONDS < 86400:
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
raise RuntimeError("Database timeouts must be positive")
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
raise RuntimeError("Database pool settings must be positive")
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
raise RuntimeError("HTTP limits must be positive")
if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024:
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB")
if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000:
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million")
if MAX_CONCURRENT_UPLOADS < 1:
raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive")
if UPLOAD_TIMEOUT_SECONDS < 1:
raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive")
+26
View File
@@ -0,0 +1,26 @@
from typing import Dict, Optional
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
class CacheUnavailableError(Exception):
pass
class IdempotencyConflictError(Exception):
pass
class AttachmentNotReadyError(Exception):
pass
class ObjectMissingError(Exception):
pass
+560
View File
@@ -0,0 +1,560 @@
import base64
import hashlib
import json
import logging
import threading
import time
import tempfile
import uuid
import warnings
import zlib
from http.server import BaseHTTPRequestHandler
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlsplit
import unicodedata
import psycopg
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, UnidentifiedImageError
from psycopg_pool import PoolTimeout
from .config import (
IDEMPOTENCY_KEY_RE,
INSTANCE_ID,
MAX_CONCURRENT_UPLOADS,
MAX_IMAGE_PIXELS,
MAX_INFLIGHT_REQUESTS,
MAX_REQUEST_BODY_BYTES,
MAX_UPLOAD_BYTES,
UPLOAD_TIMEOUT_SECONDS,
USER_RE,
)
from .errors import (
ApiError,
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .security import hash_password, isoformat, valid_uuid
from .store import Store
LOGGER = logging.getLogger("messenger.http")
class RequestGuard:
def __init__(self):
self.lock = threading.Lock()
self.inflight = 0
def acquire(self):
with self.lock:
if self.inflight >= MAX_INFLIGHT_REQUESTS:
raise ApiError(503, "unavailable", "Server is busy")
self.inflight += 1
def release(self):
with self.lock:
self.inflight = max(0, self.inflight - 1)
REQUEST_GUARD = RequestGuard()
UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1)
class MessengerHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server, store: Store):
self.store = store
super().__init__(request, client_address, server)
def setup(self):
super().setup()
self.connection.settimeout(15)
def log_message(self, format: str, *args):
return
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
acquired = False
try:
REQUEST_GUARD.acquire()
acquired = True
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query, keep_blank_values=True)
if path == "/health/live" and method == "GET":
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/health/ready" and method == "GET":
try:
self.store.ready()
except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError):
raise ApiError(503, "unavailable", "A required storage service is unavailable")
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/api/v1/users" and method == "POST":
self._handle_register(request_id)
elif path == "/api/v1/auth/sessions" and method == "POST":
self._handle_create_session(request_id)
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
self._handle_revoke_session(request_id)
elif path == "/api/v1/users/me" and method == "GET":
self._send_json(200, self._get_actor(request_id), request_id)
elif path == "/api/v1/chats" and method == "POST":
self._handle_create_chat(request_id)
elif path == "/api/v1/chats" and method == "GET":
self._handle_list_chats(query, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST":
chat_id = path.rsplit("/", 2)[1]
self._handle_upload_attachment(chat_id, query, request_id)
elif path.startswith("/api/v1/attachments/") and method == "GET":
attachment_path = path.removeprefix("/api/v1/attachments/")
if attachment_path.endswith("/content"):
attachment_id = attachment_path.removesuffix("/content")
self._handle_download_attachment(attachment_id, query, request_id)
else:
self._handle_get_attachment(attachment_path, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
chat_id = path.rsplit("/", 2)[1]
if method == "POST":
self._handle_create_message(chat_id, request_id)
else:
self._handle_list_messages(chat_id, query, request_id)
elif path.startswith("/api/v1/chats/") and method == "GET":
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
else:
raise ApiError(404, "not_found", "Object not found")
except ApiError as exc:
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
except CacheUnavailableError:
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
except IdempotencyConflictError:
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
except AttachmentNotReadyError:
self._send_error(409, "conflict", "Attachment is not ready", request_id)
except ObjectMissingError:
LOGGER.error("attachment object missing for request %s", request_id)
self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id)
except (BotoCoreError, ClientError):
LOGGER.warning("object storage unavailable for request %s", request_id)
self._send_error(503, "unavailable", "Object storage is unavailable", request_id)
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
self._send_error(503, "unavailable", "Database is unavailable", request_id)
except psycopg.errors.UniqueViolation:
self._send_error(409, "conflict", "Username already exists", request_id)
except psycopg.errors.ForeignKeyViolation:
self._send_error(400, "invalid_request", "Unknown member", request_id)
except Exception as exc:
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
self._send_error(500, "internal_error", "Internal server error", request_id)
finally:
if acquired:
REQUEST_GUARD.release()
def _read_json(self, request_id: str) -> Any:
if self.headers.get_content_type() != "application/json":
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Missing Content-Length")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size < 0:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size > MAX_REQUEST_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Request body too large")
raw = self.rfile.read(size)
if not raw:
raise ApiError(400, "invalid_request", "Empty body")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
raise ApiError(400, "invalid_request", "Malformed JSON")
def _get_actor(self, request_id: str) -> Dict[str, Any]:
header = self.headers.get("Authorization", "")
scheme, separator, token = header.partition(" ")
if scheme.lower() != "bearer" or not separator or not token or " " in token:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
user = self.store.session_user(token)
if user is None:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def _handle_register(self, request_id: str):
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
username = body["username"]
display_name = body["display_name"]
password = body["password"]
if not isinstance(username, str) or not USER_RE.fullmatch(username):
raise ApiError(400, "invalid_request", "Invalid username")
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
raise ApiError(400, "invalid_request", "Invalid display_name")
if not isinstance(password, str) or not (12 <= len(password) <= 128):
raise ApiError(400, "invalid_request", "Invalid password")
user = self.store.create_user(username, display_name, hash_password(password))
self._send_json(201, user, request_id)
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
value = self.headers.get("Authorization", "")
scheme, separator, encoded = value.partition(" ")
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
if scheme.lower() != "basic" or not separator or not encoded:
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError):
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
return username, password
def _handle_create_session(self, request_id: str):
username, password = self._basic_credentials(request_id)
user = self.store.authenticate(username, password)
if user is None:
raise ApiError(
401,
"unauthorized",
"Invalid credentials",
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
)
token, expires_at = self.store.create_session(user["id"])
self._send_json(
201,
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
request_id,
)
def _handle_revoke_session(self, request_id: str):
self._get_actor(request_id)
token = self.headers["Authorization"].partition(" ")[2]
if not self.store.revoke_session(token):
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
self.send_response(204)
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.end_headers()
def _handle_create_chat(self, request_id: str):
actor = self._get_actor(request_id)
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
raise ApiError(400, "invalid_request", "Expected title and member_ids")
title, member_ids = body["title"], body["member_ids"]
if not isinstance(title, str) or not title.strip() or len(title) > 100:
raise ApiError(400, "invalid_request", "Invalid title")
if (
not isinstance(member_ids, list)
or not 1 <= len(member_ids) <= 100
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
or len(member_ids) != len(set(member_ids))
or actor["id"] not in member_ids
):
raise ApiError(400, "invalid_request", "Invalid member_ids")
chat = self.store.create_chat(title, member_ids, actor["id"])
self._send_json(201, chat, request_id)
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
def _handle_get_chat(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat = self.store.get_chat(chat_id, actor["id"])
if chat is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(200, chat, request_id)
def _handle_create_message(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
idempotency_key = self.headers.get("Idempotency-Key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body:
raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids")
text = body["text"]
attachment_ids = body.get("attachment_ids", [])
if (
not isinstance(text, str)
or len(text) > 4000
or not isinstance(attachment_ids, list)
or len(attachment_ids) > 10
or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids)
or len(attachment_ids) != len(set(attachment_ids))
or (not text.strip() and not attachment_ids)
):
raise ApiError(400, "invalid_request", "Invalid message text")
attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids]
canonical_body = json.dumps(
{"text": text, "attachment_ids": attachment_ids},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
message = self.store.create_idempotent_message(
chat_id,
actor["id"],
text,
attachment_ids,
idempotency_key,
canonical_body,
)
if message is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(201, message, request_id)
def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]:
if declared_type not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
length = self.headers.get("Content-Length")
if length is None or self.headers.get("Transfer-Encoding"):
raise ApiError(400, "invalid_request", "A bounded Content-Length is required")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size <= 0:
raise ApiError(415, "unsupported_media_type", "Image is empty")
if size > MAX_UPLOAD_BYTES:
self.close_connection = True
remaining = min(size, MAX_UPLOAD_BYTES + 1)
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
while remaining:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
break
self.connection.settimeout(min(15, remaining_time))
try:
chunk = self.rfile.read(min(64 * 1024, remaining))
except (TimeoutError, OSError):
LOGGER.info("stopped bounded drain of oversized upload")
break
if not chunk:
break
remaining -= len(chunk)
raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit")
content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
digest = hashlib.sha256()
bytes_read = 0
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
try:
while bytes_read < size:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
raise ApiError(408, "invalid_request", "Image upload timed out")
self.connection.settimeout(min(15, remaining_time))
chunk = self.rfile.read(min(64 * 1024, size - bytes_read))
if not chunk:
raise ApiError(400, "invalid_request", "Incomplete image body")
content.write(chunk)
digest.update(chunk)
bytes_read += len(chunk)
except (TimeoutError, OSError):
content.close()
raise ApiError(400, "invalid_request", "Incomplete image body") from None
try:
content.seek(0)
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(content) as image:
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "")
if actual_type is None or actual_type != declared_type:
raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type")
width, height = image.size
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit")
image.verify()
content.seek(0)
with IMAGE_DECODE_GUARD:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(content) as image:
image.load()
content.seek(0)
return content, actual_type, size, digest.hexdigest()
except ApiError:
content.close()
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
content.close()
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error):
content.close()
raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None
@staticmethod
def _valid_filename(filename: str) -> bool:
return (
1 <= len(filename) <= 255
and not any(char in ("/", "\\", "\x00") for char in filename)
and not any(unicodedata.category(char) == "Cc" for char in filename)
)
def _handle_upload_attachment(
self, chat_id: str, query: Dict[str, List[str]], request_id: str
):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
if not self.store.can_access_chat(chat_id, actor["id"]):
raise ApiError(404, "not_found", "Chat not found")
filenames = query.get("filename", [])
if len(filenames) != 1 or not self._valid_filename(filenames[0]):
raise ApiError(400, "invalid_request", "Invalid filename")
if not UPLOAD_GUARD.acquire(blocking=False):
raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted")
try:
content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type())
try:
attachment = self.store.create_attachment(
chat_id,
actor["id"],
filenames[0],
content_type,
content,
size,
digest,
)
finally:
content.close()
finally:
UPLOAD_GUARD.release()
if attachment is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(202, attachment, request_id)
def _handle_get_attachment(self, attachment_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(attachment_id):
raise ApiError(400, "invalid_request", "Invalid attachment id")
attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"])
if attachment is None:
raise ApiError(404, "not_found", "Attachment not found")
self._send_json(200, attachment, request_id)
def _handle_download_attachment(
self, attachment_id: str, query: Dict[str, List[str]], request_id: str
):
actor = self._get_actor(request_id)
if not valid_uuid(attachment_id):
raise ApiError(400, "invalid_request", "Invalid attachment id")
variants = query.get("variant", ["original"])
if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"):
raise ApiError(400, "invalid_request", "Invalid attachment variant")
variant = variants[0]
if variant != "original" and variant not in ("thumbnail", "crop", "optimized"):
raise ApiError(404, "not_found", "Attachment variant not found")
result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant)
if result is None:
raise ApiError(404, "not_found", "Attachment not found")
content_type, content = result
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(content)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Cache-Control", "private, no-store")
self.end_headers()
self.wfile.write(content)
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if result is None:
raise ApiError(404, "not_found", "Chat not found")
items, next_cursor = result
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
def _send_json(
self,
status: int,
payload: Dict[str, Any],
request_id: str,
headers: Optional[Dict[str, str]] = None,
):
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
if headers:
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body)
def _send_error(
self,
status: int,
code: str,
message: str,
request_id: str,
extra_headers: Optional[Dict[str, str]] = None,
):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
+50
View File
@@ -0,0 +1,50 @@
import json
import os
from pathlib import Path
import pika
QUEUE_NAME = "attachments.process.v1"
DEAD_QUEUE_NAME = "attachments.process.dead"
LEASE_NETWORK_BUDGET_SECONDS = 150
def broker_url() -> str:
value = os.getenv("BROKER_URL")
if value:
return value
file_path = os.getenv("BROKER_URL_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError("Set BROKER_URL or BROKER_URL_FILE")
def connect() -> pika.BlockingConnection:
parameters = pika.URLParameters(broker_url())
parameters.connection_attempts = 3
parameters.retry_delay = 1
parameters.socket_timeout = 5
parameters.blocked_connection_timeout = 10
return pika.BlockingConnection(parameters)
def declare_queues(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
channel.queue_declare(
queue=DEAD_QUEUE_NAME,
durable=True,
arguments={"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
)
channel.queue_declare(
queue=QUEUE_NAME,
durable=True,
arguments={
"x-queue-type": "quorum",
"x-quorum-initial-group-size": 1,
"x-dead-letter-exchange": "",
"x-dead-letter-routing-key": DEAD_QUEUE_NAME,
},
)
def encode_job(job: dict) -> bytes:
return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8")
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
import logging
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
def main():
logging.basicConfig(level="INFO")
store = Store()
store.open()
try:
store.migrate()
finally:
store.close()
if __name__ == "__main__":
main()
+76
View File
@@ -0,0 +1,76 @@
import logging
from datetime import datetime
from typing import BinaryIO, Iterator, Optional, Tuple, Union
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY
from .errors import ObjectMissingError
LOGGER = logging.getLogger("messenger.s3")
class ObjectStore:
def __init__(self):
self.client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
config=Config(
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
),
)
def put(
self,
key: str,
content: Union[bytes, BinaryIO],
content_type: str,
size_bytes: Optional[int] = None,
) -> None:
if size_bytes is None:
size_bytes = len(content)
self.client.put_object(
Bucket=S3_BUCKET,
Key=key,
Body=content,
ContentLength=size_bytes,
ContentType=content_type,
)
def get(self, key: str, max_bytes: int) -> bytes:
try:
response = self.client.get_object(Bucket=S3_BUCKET, Key=key)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code in ("NoSuchKey", "404", "NotFound"):
raise ObjectMissingError from exc
raise
body = response["Body"]
try:
value = body.read(max_bytes + 1)
finally:
body.close()
if len(value) > max_bytes:
raise RuntimeError("Stored object exceeds configured safety limit")
return value
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=S3_BUCKET, Key=key)
def ready(self) -> bool:
self.client.head_bucket(Bucket=S3_BUCKET)
return True
def objects(self) -> Iterator[Tuple[str, datetime]]:
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET)
for page in pages:
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"]
+4
View File
@@ -0,0 +1,4 @@
PIPELINE_QUALITY = {
1: {"jpeg": 85, "webp": 80},
2: {"jpeg": 80, "webp": 75},
}
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
import logging
import os
import time
import uuid
import pika
import psycopg
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues, encode_job
from .security import isoformat
from .store import Store
LOGGER = logging.getLogger("messenger.publisher")
def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
with store.pool.connection() as conn:
with conn.transaction():
row = store.claim_outbox(conn)
if row is None:
return False
payload = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"created_at": isoformat(row["created_at"]),
}
channel.basic_publish(
exchange="",
routing_key=QUEUE_NAME,
body=encode_job(payload),
properties=pika.BasicProperties(
content_type="application/json",
delivery_mode=2,
message_id=str(row["id"]),
type="attachment.process.v1",
),
mandatory=True,
)
conn.execute(
"UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
return True
def main() -> None:
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
store = Store()
store.open()
try:
while True:
connection = None
try:
connection = connect()
channel = connection.channel()
declare_queues(channel)
channel.confirm_delivery()
store.recover_expired_attachment_jobs(
int(os.getenv("JOB_MAX_ATTEMPTS", "5")),
int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS,
)
if not publish_pending(store, channel):
time.sleep(1)
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
LOGGER.exception("outbox publishing cycle failed")
time.sleep(2)
finally:
if connection is not None and connection.is_open:
connection.close()
finally:
store.close()
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import logging
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
LOGGER = logging.getLogger("messenger.reconcile")
def reconcile(
store: Store,
grace_seconds: int,
apply_changes: bool,
restore_dir: Optional[Path] = None,
):
started = time.monotonic()
metadata = store.reconciliation_snapshot()
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
found = set()
orphans = []
for key, last_modified in store.objects.objects():
found.add(key)
if key not in metadata and last_modified <= cutoff:
orphans.append(key)
missing = [
row for key, row in metadata.items()
if key not in found and row.get("reconcile_type", "original") == "original"
]
restore_candidates = []
if restore_dir is not None:
root = restore_dir.resolve(strict=True)
for row in missing:
source = restore_dir / str(row["id"])
try:
resolved = source.resolve(strict=True)
if (
resolved.parent != root
or not resolved.is_file()
or resolved.stat().st_size != row["size_bytes"]
):
continue
content = resolved.read_bytes()
except OSError:
continue
if (
len(content) == row["size_bytes"]
and hashlib.sha256(content).hexdigest() == row["sha256"].strip()
):
restore_candidates.append((row, content))
restored = 0
if apply_changes:
for key in orphans:
store.objects.delete(key)
restorable_ids = set()
for row, content in restore_candidates:
store.objects.put(row["object_key"], content, row["content_type"])
store.clear_missing_object(row["id"])
restorable_ids.add(row["id"])
restored += 1
for row in missing:
if row["id"] not in restorable_ids:
store.record_missing_object(row["id"])
for key, row in metadata.items():
if key in found and row.get("reconcile_type", "original") == "original":
store.clear_missing_object(row["id"])
unresolved_missing = len(missing) - restored
return {
"mode": "apply" if apply_changes else "dry-run",
"grace_seconds": grace_seconds,
"objects_seen": len(found),
"metadata_seen": len({str(row["id"]) for row in metadata.values()}),
"orphan_objects": len([key for key in found if key not in metadata]),
"stale_orphan_candidates": len(orphans),
"deleted_orphans": len(orphans) if apply_changes else 0,
"missing_objects": len(missing),
"unresolved_missing_objects": unresolved_missing,
"restorable_missing_objects": len(restore_candidates),
"restored_missing_objects": restored,
"missing_incidents_recorded": len(missing) - restored if apply_changes else 0,
"duration_ms": round((time.monotonic() - started) * 1000, 3),
}
def main():
parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.")
parser.add_argument("--grace-seconds", type=int, default=3600)
parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects")
parser.add_argument(
"--restore-dir",
type=Path,
help="optional trusted backup directory with exact original bytes named by attachment UUID",
)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error("--grace-seconds must be at least 60")
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
store = Store()
store.open()
try:
result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir)
print(json.dumps(result, sort_keys=True))
if result["unresolved_missing_objects"]:
LOGGER.error(
"%s database records reference missing S3 objects; metadata was retained for manual recovery",
result["unresolved_missing_objects"],
)
finally:
store.close()
if __name__ == "__main__":
main()
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
import argparse
import json
import uuid
from .store import Store
def main() -> None:
parser = argparse.ArgumentParser(
description="Explicitly replay a failed attachment or start a newer pipeline version."
)
parser.add_argument("--attachment-id", required=True, type=uuid.UUID)
parser.add_argument(
"--pipeline-version",
required=True,
type=int,
help="pipeline version implemented by the deployed worker",
)
args = parser.parse_args()
if args.pipeline_version < 1:
parser.error("--pipeline-version must be positive")
store = Store()
store.open()
try:
result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version)
print(json.dumps(result, sort_keys=True))
finally:
store.close()
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
import base64
import hashlib
import hmac
import secrets
import uuid
from datetime import datetime, timezone
def isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def valid_uuid(value: str) -> bool:
try:
uuid.UUID(value)
return True
except (ValueError, TypeError, AttributeError):
return False
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
derived = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
)
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
return f"scrypt$32768$8$1${salt_text}${hash_text}"
def verify_password(password: str, stored: str) -> bool:
try:
scheme, n, r, p, salt_text, hash_text = stored.split("$")
if scheme != "scrypt":
return False
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
actual = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=int(n),
r=int(r),
p=int(p),
dklen=len(expected),
maxmem=64 * 1024 * 1024,
)
return hmac.compare_digest(actual, expected)
except (ValueError, TypeError, UnicodeError):
return False
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.app import main
else:
from .app import main
if __name__ == "__main__":
main()
+835
View File
@@ -0,0 +1,835 @@
import base64
import hashlib
import hmac
import json
import logging
import math
import os
import secrets
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
import redis
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import (
DB_LOCK_TIMEOUT_MS,
DB_POOL_MAX_SIZE,
DB_POOL_TIMEOUT_SECONDS,
DB_STATEMENT_TIMEOUT_MS,
IDEMPOTENCY_TTL_SECONDS,
SESSION_TTL_SECONDS,
)
from .errors import (
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .object_store import ObjectStore
from .pipeline_versions import PIPELINE_QUALITY
from .security import isoformat, utc_now, valid_uuid, verify_password
LOGGER = logging.getLogger("messenger.store")
def database_url() -> str:
value = os.getenv("DATABASE_URL")
if value:
return value
file_path = os.getenv("DATABASE_URL_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
class Store:
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
def __init__(self):
self.pool = ConnectionPool(
conninfo=database_url(),
min_size=1,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": (
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
"-c idle_in_transaction_session_timeout=10000"
),
},
open=False,
)
cache_url = os.getenv("CACHE_URL")
self.cache = (
redis.Redis.from_url(
cache_url,
decode_responses=True,
socket_connect_timeout=0.25,
socket_timeout=0.25,
retry_on_timeout=False,
max_connections=16,
)
if cache_url
else None
)
self.objects = ObjectStore()
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
if self.cache is not None:
self.cache.close()
self.pool.close()
def ready(self) -> bool:
with self.pool.connection(timeout=2) as conn:
conn.execute("SELECT 1")
self.objects.ready()
return True
def migrate(self):
migrations = Path(__file__).resolve().parents[1] / "migrations"
with self.pool.connection() as conn:
with conn.transaction():
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
for migration in sorted(migrations.glob("*.sql")):
if migration.name in applied:
continue
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
@staticmethod
def user(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": isoformat(row["created_at"]),
"attachment_ids": attachment_ids or [],
}
@staticmethod
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
variants = row.get("variants") or []
public_variants = [
{
name: variant[name]
for name in ("name", "content_type", "size_bytes", "width", "height", "sha256")
}
for variant in variants
]
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"].strip(),
"status": row.get("status", "queued"),
"variants": public_variants,
"error_code": row.get("error_code"),
"created_at": isoformat(row["created_at"]),
"updated_at": isoformat(row["updated_at"]),
}
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO users (username, display_name, password_hash) "
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
(username, display_name, password_hash),
).fetchone()
return self.user(row)
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
return self.user(row) if row else None
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at, password_hash "
"FROM users WHERE username = %s",
(username,),
).fetchone()
if not row or not verify_password(password, row["password_hash"]):
return None
return self.user(row)
def create_session(self, user_id: str) -> Tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
token_hash = hashlib.sha256(token.encode("ascii")).digest()
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, user_id, expires_at),
)
return token, expires_at
@staticmethod
def _session_lock_key(token_hash: bytes) -> int:
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
cache_key = "messenger:session:" + token_hash.hex()
if self.cache is not None:
try:
cached = self.cache.get(cache_key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError, TypeError):
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
row = conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
"FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE s.token_hash = %s AND s.expires_at > now()",
(token_hash,),
).fetchone()
if not row:
return None
user = self.user(row)
if self.cache is not None:
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
if ttl_ms > 0:
try:
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
except redis.RedisError:
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
return user
def revoke_session(self, token: str) -> bool:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
if self.cache is not None:
try:
self.cache.delete("messenger:session:" + token_hash.hex())
except redis.RedisError as exc:
raise CacheUnavailableError from exc
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
return result.rowcount == 1
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
raise ValueError("invalid members")
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
(title,),
).fetchone()
for member_id in member_ids:
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
(row["id"], member_id),
)
return self.chat(row, member_ids)
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id "
"WHERE c.id = %s AND m.user_id = %s",
(chat_id, user_id),
).fetchone()
if not row:
return None
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(chat_id,),
).fetchall()
return self.chat(row, [str(item["user_id"]) for item in members])
def list_chats(
self, user_id: str, limit: int, cursor: Optional[str]
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
where = "own.user_id = %s"
params: List[Any] = [user_id]
if cursor is not None:
try:
raw = base64.b64decode(
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
altchars=b"-_",
validate=True,
)
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
if cursor_user != user_id or not valid_uuid(chat_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if created_at.tzinfo is None or created_at.utcoffset() is None:
raise ValueError("cursor timestamp must include a timezone")
where += " AND (c.created_at, c.id) > (%s, %s)"
params.extend((created_at, chat_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members own ON own.chat_id = c.id "
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
result = []
for row in page:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(row["id"],),
).fetchall()
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return result, next_cursor
def create_idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
attachment_ids: List[str],
idempotency_key: str,
canonical_body: str,
) -> Optional[Dict[str, Any]]:
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
with self.pool.connection() as conn:
with conn.transaction():
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if not member:
return None
if attachment_ids:
attachments = conn.execute(
"SELECT id, status FROM attachments "
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
(chat_id, attachment_ids),
).fetchall()
if len(attachments) != len(attachment_ids):
return None
if any(item["status"] == "failed" for item in attachments):
raise AttachmentNotReadyError
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
existing = conn.execute(
"SELECT request_hash, response, expires_at "
"FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
"FOR UPDATE",
(sender_id, chat_id, idempotency_key),
).fetchone()
if existing and existing["expires_at"] > utc_now():
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
raise IdempotencyConflictError
return existing["response"]
if existing:
conn.execute(
"DELETE FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
(sender_id, chat_id, idempotency_key),
)
row = conn.execute(
"INSERT INTO messages (chat_id, sender_id, text) "
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
(chat_id, sender_id, text),
).fetchone()
for ordinal, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
"VALUES (%s, %s, %s)",
(row["id"], attachment_id, ordinal),
)
message = self.message(row, attachment_ids)
conn.execute(
"INSERT INTO idempotency_records "
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
(
sender_id,
chat_id,
idempotency_key,
request_hash,
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
),
)
return message
def list_messages(
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
with self.pool.connection() as conn:
is_member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if not is_member:
return None
where = "chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
if decoded_chat != chat_id or not valid_uuid(message_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
where += " AND (created_at, id) > (%s, %s)"
params.extend((created_at, message_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
rows = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
params,
).fetchall()
ids_by_message = {}
if rows:
links = conn.execute(
"SELECT message_id, attachment_id FROM message_attachments "
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
([row["id"] for row in rows],),
).fetchall()
for link in links:
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
with self.pool.connection() as conn:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone() is not None
def create_attachment(
self,
chat_id: str,
owner_id: str,
filename: str,
content_type: str,
content: BinaryIO,
size_bytes: int,
digest: str,
) -> Optional[Dict[str, Any]]:
if not self.can_access_chat(chat_id, owner_id):
return None
attachment_id = uuid.uuid4()
job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}"
object_key = f"original/{attachment_id}"
content.seek(0)
self.objects.put(staging_key, content, content_type, size_bytes)
try:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO attachments "
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"object_key, staging_key, job_id) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, variants, error_code, created_at, updated_at",
(
attachment_id,
chat_id,
owner_id,
filename,
content_type,
size_bytes,
digest,
object_key,
staging_key,
job_id,
),
).fetchone()
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, 1, 1)",
(uuid.uuid4(), job_id, attachment_id),
)
return self.attachment(row)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"attachment staging compensation failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
"a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
return self.attachment(row) if row else None
def download_attachment(
self, attachment_id: str, user_id: str, variant: str = "original"
) -> Optional[Tuple[str, bytes]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
if not row:
return None
if row["status"] != "ready":
raise AttachmentNotReadyError
if variant == "original":
key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"]
else:
match = next(
(item for item in (row["variants"] or []) if item["name"] == variant),
None,
)
if match is None:
return None
key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"]
return content_type, self.objects.get(key, size_bytes)
def claim_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int
) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, "
"object_key, content_type, size_bytes, sha256 "
"FROM attachments WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
not row
or str(row["job_id"]) != job_id
or row["pipeline_version"] != pipeline_version
or row["status"] in ("ready", "failed")
or row["attempt"] != attempt
):
return None
if row["status"] == "processing" and row["lease_until"] is not None:
if row["lease_until"] > utc_now():
return None
return conn.execute(
"UPDATE attachments SET status = 'processing', "
"lease_until = clock_timestamp() + (%s * interval '1 second'), "
"updated_at = clock_timestamp() "
"WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, "
"size_bytes, sha256, attempt, pipeline_version",
(lease_seconds, attachment_id),
).fetchone()
def complete_attachment_job(
self,
attachment_id: str,
job_id: str,
pipeline_version: int,
attempt: int,
variants: List[Dict[str, Any]],
) -> bool:
with self.pool.connection() as conn:
with conn.transaction():
updated = conn.execute(
"UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, "
"lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s "
"AND status = 'processing' AND lease_until > clock_timestamp()",
(
json.dumps(variants, separators=(",", ":")),
attachment_id,
job_id,
pipeline_version,
attempt,
),
)
return updated.rowcount == 1
def fail_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int,
max_attempts: int, error_code: str, retry_delay_seconds: Optional[float],
) -> None:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT status, attempt FROM attachments "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE",
(attachment_id, job_id, pipeline_version),
).fetchone()
if not row or row["status"] != "processing" or row["attempt"] != attempt:
return
if retry_delay_seconds is not None and attempt < max_attempts:
next_attempt = attempt + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, error_code, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, available_at) "
"VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second')) "
"ON CONFLICT (job_id, attempt) DO NOTHING",
(
uuid.uuid4(), job_id, attachment_id, pipeline_version,
next_attempt, retry_delay_seconds,
),
)
else:
final_code = "retry_exhausted" if attempt >= max_attempts else error_code
conn.execute(
"UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, "
"updated_at = clock_timestamp() WHERE id = %s",
(final_code, attachment_id),
)
def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]:
if pipeline_version not in PIPELINE_QUALITY:
raise ValueError("pipeline version is not implemented by this deployment")
with self.pool.connection() as conn:
current = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, "
"content_type, size_bytes, sha256 FROM attachments WHERE id = %s",
(attachment_id,),
).fetchone()
if current is None:
raise ValueError("attachment not found")
if pipeline_version < current["pipeline_version"]:
raise ValueError("pipeline version cannot move backwards")
if pipeline_version == current["pipeline_version"] and current["status"] != "failed":
raise ValueError("same-version replay is allowed only for failed attachments")
if current["status"] not in ("queued", "processing", "ready", "failed"):
raise ValueError("attachment cannot be replayed in its current state")
try:
content = self.objects.get(current["object_key"], current["size_bytes"])
except ObjectMissingError as object_error:
try:
content = self.objects.get(current["staging_key"], current["size_bytes"])
except ObjectMissingError:
raise object_error
if (
len(content) != current["size_bytes"]
or hashlib.sha256(content).hexdigest() != current["sha256"].strip()
):
raise ValueError("stored original does not match attachment metadata")
prior_job_id = str(current["job_id"])
next_job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}/{next_job_id}"
self.objects.put(staging_key, content, current["content_type"], len(content))
try:
with self.pool.connection() as conn:
with conn.transaction():
locked = conn.execute(
"SELECT job_id, pipeline_version, attempt, status FROM attachments "
"WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
locked is None
or str(locked["job_id"]) != prior_job_id
or locked["pipeline_version"] != current["pipeline_version"]
or locked["attempt"] != current["attempt"]
or locked["status"] != current["status"]
):
raise RuntimeError("attachment changed while preparing replay")
conn.execute(
"UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, "
"status = 'queued', staging_key = %s, variants = '[]'::jsonb, "
"error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s",
(next_job_id, pipeline_version, staging_key, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, 1)",
(uuid.uuid4(), next_job_id, attachment_id, pipeline_version),
)
conn.execute(
"INSERT INTO attachment_pipeline_events "
"(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) "
"VALUES (%s, %s, %s, %s, %s, 'replay')",
(
attachment_id,
prior_job_id,
next_job_id,
current["pipeline_version"],
pipeline_version,
),
)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"replay staging cleanup failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
return {
"attachment_id": str(attachment_id),
"previous_job_id": prior_job_id,
"job_id": str(next_job_id),
"pipeline_version": pipeline_version,
"attempt": 1,
"status": "queued",
}
def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int:
with self.pool.connection() as conn:
with conn.transaction():
rows = conn.execute(
"SELECT id, job_id, pipeline_version, attempt FROM attachments "
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
"ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100"
).fetchall()
for row in rows:
if row["attempt"] >= max_attempts:
conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', "
"lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
continue
next_attempt = row["attempt"] + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, row["id"]),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING",
(uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt),
)
return len(rows)
def claim_outbox(self, conn):
return conn.execute(
"SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, "
"a.created_at FROM attachment_outbox o "
"JOIN attachments a ON a.id = o.attachment_id "
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
"ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1"
).fetchone()
def reconciliation_snapshot(self):
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, "
"size_bytes, sha256, content_type FROM attachments"
).fetchall()
expected = {}
for row in rows:
if row["status"] == "ready":
expected[row["object_key"]] = {**row, "reconcile_type": "original"}
for variant in row["variants"] or []:
expected[variant["object_key"]] = {
**row,
"object_key": variant["object_key"],
"size_bytes": variant["size_bytes"],
"sha256": variant["sha256"],
"content_type": variant["content_type"],
"reconcile_type": "variant",
}
elif row["status"] in ("queued", "processing"):
expected[row["staging_key"]] = {**row, "reconcile_type": "staging"}
expected[row["object_key"]] = {**row, "reconcile_type": "inflight"}
for name, extension in (
("thumbnail", "jpg"),
("crop", "jpg"),
("optimized", "webp"),
):
key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}"
expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"}
return expected
def record_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
"DO UPDATE SET last_seen_at = clock_timestamp()",
(attachment_id,),
)
def clear_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"DELETE FROM reconciliation_incidents "
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
(attachment_id,),
)
+341
View File
@@ -0,0 +1,341 @@
#!/usr/bin/env python3
import hashlib
import io
import json
import logging
import multiprocessing
import os
import random
import time
import uuid
import warnings
import zlib
from typing import Callable, Optional
import pika
import psycopg
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, ImageOps, UnidentifiedImageError
from .config import MAX_IMAGE_PIXELS
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues
from .pipeline_versions import PIPELINE_QUALITY
from .store import Store
LOGGER = logging.getLogger("messenger.worker")
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
class InvalidAttachment(Exception):
pass
class UnsupportedPipeline(InvalidAttachment):
pass
def _rgb_image(content: bytes) -> Image.Image:
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(content)) as source:
if source.format not in ("PNG", "JPEG"):
raise InvalidAttachment("unsupported_image")
width, height = source.size
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
raise InvalidAttachment("image_too_large")
source.load()
image = ImageOps.exif_transpose(source)
try:
if "A" in image.getbands() or "transparency" in image.info:
with image.convert("RGBA") as rgba:
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return image.convert("RGB")
finally:
if image is not source:
image.close()
except InvalidAttachment:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError,
UnidentifiedImageError, ValueError, zlib.error) as exc:
raise InvalidAttachment("invalid_image") from exc
def _encode(image: Image.Image, image_format: str, **options) -> bytes:
output = io.BytesIO()
image.save(output, format=image_format, **options)
return output.getvalue()
def _resize_within(image: Image.Image, bounds: tuple[int, int]) -> Image.Image:
width, height = image.size
scale = min(1.0, bounds[0] / width, bounds[1] / height)
size = (max(1, int(width * scale)), max(1, int(height * scale)))
if size == image.size:
return image.copy()
return image.resize(size, Image.Resampling.LANCZOS)
def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1):
quality = PIPELINE_QUALITY.get(pipeline_version)
if quality is None:
raise UnsupportedPipeline("unsupported_pipeline_version")
image = _rgb_image(content)
try:
variants = []
thumbnail = _resize_within(image, (256, 256))
try:
thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True)
variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes))
finally:
thumbnail.close()
crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
try:
crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True)
variants.append(("crop", "image/jpeg", 128, 128, crop_bytes))
finally:
crop.close()
optimized = _resize_within(image, (1280, 1280))
try:
optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6)
variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes))
finally:
optimized.close()
result = []
for name, content_type, width, height, data in variants:
extension = "webp" if content_type == "image/webp" else "jpg"
key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}"
result.append(
{
"name": name,
"content_type": content_type,
"size_bytes": len(data),
"width": width,
"height": height,
"sha256": hashlib.sha256(data).hexdigest(),
"object_key": key,
"content": data,
}
)
return result
finally:
image.close()
class ProcessingTimeout(TimeoutError):
pass
def _variant_process(connection, attachment_id, content, pipeline_version, builder):
try:
connection.send(("ok", builder(attachment_id, content, pipeline_version)))
except InvalidAttachment as exc:
connection.send(("invalid", str(exc)))
except Exception as exc:
connection.send(("error", type(exc).__name__))
finally:
connection.close()
def _build_variants_bounded(
attachment_id: str,
content: bytes,
pipeline_version: int,
timeout_seconds: int,
builder=build_variants,
):
if timeout_seconds < 1:
raise ValueError("timeout_seconds must be positive")
context = multiprocessing.get_context("spawn")
receiver, sender = context.Pipe(duplex=False)
process = context.Process(
target=_variant_process,
args=(sender, attachment_id, content, pipeline_version, builder),
)
process.start()
sender.close()
try:
if not receiver.poll(timeout_seconds):
process.terminate()
process.join(timeout=2)
if process.is_alive():
process.kill()
process.join()
raise ProcessingTimeout("image processing exceeded its time budget")
try:
result, value = receiver.recv()
except EOFError as exc:
raise RuntimeError("image processing process exited without a result") from exc
process.join(timeout=2)
if process.is_alive():
process.terminate()
process.join()
if result == "invalid":
raise InvalidAttachment(value)
if result == "error":
raise RuntimeError(f"image processing failed in subprocess ({value})")
return value
finally:
receiver.close()
if process.is_alive():
process.terminate()
process.join()
def _job(payload: bytes):
value = json.loads(payload)
required = {
"schema_version", "job_id", "type", "attachment_id",
"pipeline_version", "attempt", "created_at",
}
if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}):
raise ValueError("invalid job fields")
if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1":
raise ValueError("invalid job schema")
uuid.UUID(value["job_id"])
uuid.UUID(value["attachment_id"])
if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1:
raise ValueError("invalid pipeline version")
if not isinstance(value["attempt"], int) or value["attempt"] < 1:
raise ValueError("invalid job attempt")
return value
def _process(
store: Store,
job: dict,
fault_injector: Optional[Callable[[str], None]] = None,
) -> None:
def inject(point: str) -> None:
if fault_injector is not None:
fault_injector(point)
attachment_id = job["attachment_id"]
row = store.claim_attachment_job(
attachment_id,
job["job_id"],
job["pipeline_version"],
job["attempt"],
JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS,
)
if row is None:
LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"])
return
try:
inject("after_claim")
content = store.objects.get(row["staging_key"], row["size_bytes"])
if hashlib.sha256(content).hexdigest() != row["sha256"].strip():
raise InvalidAttachment("checksum_mismatch")
variants = _build_variants_bounded(
attachment_id,
content,
job["pipeline_version"],
JOB_TIMEOUT_SECONDS,
)
inject("before_original_put")
store.objects.put(row["object_key"], content, row["content_type"], len(content))
inject("after_original_put")
persisted = []
for variant in variants:
data = variant.pop("content")
inject(f"before_variant_put:{variant['name']}")
store.objects.put(variant["object_key"], data, variant["content_type"], len(data))
inject(f"after_variant_put:{variant['name']}")
persisted.append(variant)
inject("before_commit")
if not store.complete_attachment_job(
attachment_id,
job["job_id"],
job["pipeline_version"],
job["attempt"],
persisted,
):
LOGGER.warning("job lease was superseded before completion: %s", job["job_id"])
return
inject("after_commit")
try:
store.objects.delete(row["staging_key"])
except (BotoCoreError, ClientError, OSError):
LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id)
LOGGER.info("attachment processing completed: %s", attachment_id)
except InvalidAttachment as exc:
store.fail_attachment_job(
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
MAX_ATTEMPTS, str(exc), None,
)
try:
store.objects.delete(row["staging_key"])
except (BotoCoreError, ClientError, OSError):
LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id)
LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc)
except Exception as exc:
delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25)
store.fail_attachment_job(
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
MAX_ATTEMPTS, "processing_failed", delay,
)
LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__)
def handle_delivery(
store: Store,
channel: pika.adapters.blocking_connection.BlockingChannel,
method,
body: bytes,
fault_injector: Optional[Callable[[str], None]] = None,
) -> None:
try:
job = _job(body)
except (ValueError, TypeError, json.JSONDecodeError):
LOGGER.exception("invalid broker job rejected")
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
return
_process(store, job, fault_injector)
if fault_injector is not None:
fault_injector("before_ack")
channel.basic_ack(delivery_tag=method.delivery_tag)
def main() -> None:
if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1:
raise RuntimeError("Worker limits must be positive")
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
store = Store()
store.open()
connection = None
try:
while True:
try:
connection = connect()
channel = connection.channel()
declare_queues(channel)
channel.basic_qos(prefetch_count=WORKER_CONCURRENCY)
def on_message(ch, method, properties, body):
handle_delivery(store, ch, method, body)
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False)
LOGGER.info("worker consuming durable attachment queue")
channel.start_consuming()
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
LOGGER.exception("worker connection failed")
time.sleep(2)
finally:
if connection is not None and connection.is_open:
connection.close()
connection = None
finally:
store.close()
if __name__ == "__main__":
main()
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Negative attachment and cross-chat authorization checks against a running lab4 API."""
import uuid
from pathlib import Path
import sys
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202):
return client.call(
"POST",
"/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""),
expected,
actor=actor,
raw=data,
headers={"Content-Type": content_type},
schema="Attachment" if expected in (201, 202) else None,
)
def main():
args = arguments("Attachment upload, size, MIME, and chat authorization checks")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
other_chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]},
schema="Chat",
)["json"]
image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
print("Checking accepted fixture upload.", flush=True)
attachment = upload(client, alice, chat["id"], image)["json"]
print("Checking damaged and spoofed image rejection.", flush=True)
upload(client, alice, chat["id"], b"not an image", expected=415)
upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415)
print("Checking byte and pixel limits.", flush=True)
upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413)
import struct
import zlib
def chunk(kind, payload):
return (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
)
bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30"))
bad_idat[-3] ^= 0xFF
damaged_idat = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", bytes(bad_idat))
+ chunk(b"IEND", b"")
)
upload(client, alice, chat["id"], damaged_idat, expected=415)
too_many_pixels = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(b""))
+ chunk(b"IEND", b"")
)
upload(client, alice, chat["id"], too_many_pixels, expected=413)
print("Checking filename and cross-chat authorization.", flush=True)
upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400)
upload(client, eve, chat["id"], image, expected=404)
client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve)
client.call(
"POST",
"/api/v1/chats/" + other_chat["id"] + "/messages",
404,
actor=alice,
headers={"Idempotency-Key": uuid.uuid4().hex},
body={"text": "", "attachment_ids": [attachment["id"]]},
)
client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
400,
actor=bob,
headers={"Idempotency-Key": uuid.uuid4().hex},
body={"text": ""},
)
print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases")
if __name__ == "__main__":
run(main)
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Prove a concurrent idempotency-key race through the load balancer."""
from concurrent.futures import ThreadPoolExecutor
import threading
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def main():
args = arguments("Concurrent shared-idempotency integration check")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = "race-" + uuid.uuid4().hex
text = "concurrent-" + uuid.uuid4().hex
count = 12
barrier = threading.Barrier(count)
def send_once(_):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=10)
return worker.message(alice, chat, text, key=key)
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(send_once, range(count)))
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
messages = [response["json"] for response in responses]
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]
matches = [message for message in page["items"] if message["text"] == text]
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
if __name__ == "__main__":
run(main)
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Verify an upload committed while the publisher is stopped is processed after restart."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments("Verify committed attachment outbox intent survives publisher downtime")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
compose("stop", "publisher")
try:
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not accepted into queued state")
finally:
compose("start", "publisher")
ready = client.wait_ready(alice, attachment)
stored_original = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=alice,
)["bytes"]
thumbnail = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
variant = next(item for item in ready["variants"] if item["name"] == "thumbnail")
check(stored_original == original, "recovered job changed original bytes")
check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch")
print(
json.dumps(
{
"result": "PASS",
"accepted_status_before_publisher_restart": attachment["status"],
"final_status": ready["status"],
"attachment_id": attachment["id"],
"original_sha256": hashlib.sha256(stored_original).hexdigest(),
"thumbnail_sha256": variant["sha256"],
"recovered_without_manual_republish": True,
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+485
View File
@@ -0,0 +1,485 @@
#!/usr/bin/env python3
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
import hashlib
import json
import os
from pathlib import Path
import sys
import time
import uuid
from datetime import datetime, timezone
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import pika
from src.publisher import publish_pending
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
JOB_ID = "00000000-0000-4000-8000-000000000002"
class SimulatedCrash(BaseException):
pass
class FakeObjects:
def __init__(self, content):
self.data = {"staging/" + ATTACHMENT_ID: content}
self.puts = []
def get(self, key, max_bytes):
return self.data[key]
def put(self, key, content, content_type, size_bytes=None):
value = content if isinstance(content, bytes) else content.read()
assert len(value) == size_bytes
self.data[key] = value
self.puts.append((key, hashlib.sha256(value).hexdigest()))
def delete(self, key):
self.data.pop(key, None)
class FakeStore:
def __init__(self, content):
self.objects = FakeObjects(content)
self.row = {
"id": ATTACHMENT_ID,
"job_id": JOB_ID,
"pipeline_version": 1,
"attempt": 1,
"status": "queued",
"lease_until": 0.0,
"staging_key": "staging/" + ATTACHMENT_ID,
"object_key": "original/" + ATTACHMENT_ID,
"content_type": "image/png",
"size_bytes": len(content),
"sha256": hashlib.sha256(content).hexdigest(),
"variants": [],
}
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] in ("ready", "failed")
):
return None
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
return None
row["status"] = "processing"
row["lease_until"] = time.monotonic() + lease_seconds
return dict(row)
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] != "processing"
or row["lease_until"] <= time.monotonic()
):
return False
row["status"] = "ready"
row["lease_until"] = 0.0
row["variants"] = variants
return True
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
row = self.row
if (
row["id"] == attachment_id
and row["job_id"] == job_id
and row["pipeline_version"] == version
and row["attempt"] == attempt
and row["status"] == "processing"
):
row["status"] = "failed" if delay is None else "queued"
row["lease_until"] = 0.0
row["error_code"] = error_code
if delay is not None:
row["attempt"] += 1
def recover_expired(self):
if self.row["status"] == "processing":
self.row["attempt"] += 1
self.row["status"] = "queued"
self.row["lease_until"] = 0.0
def replay(self, version):
row = self.row
previous_job = row["job_id"]
row["job_id"] = str(uuid.uuid4())
row["pipeline_version"] = version
row["attempt"] = 1
row["status"] = "queued"
row["lease_until"] = 0.0
row["variants"] = []
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
self.objects.put(
row["staging_key"],
self.objects.data[row["object_key"]],
row["content_type"],
row["size_bytes"],
)
return {
"schema_version": 1,
"job_id": row["job_id"],
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
"previous_job_id": previous_job,
}
class FakeChannel:
def __init__(self):
self.acked = []
self.nacked = []
def basic_ack(self, delivery_tag):
self.acked.append(delivery_tag)
def basic_nack(self, delivery_tag, requeue):
self.nacked.append((delivery_tag, requeue))
class FakeMethod:
def __init__(self, delivery_tag):
self.delivery_tag = delivery_tag
def job_body(job_id=JOB_ID, version=1, attempt=1):
return json.dumps(
{
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": attempt,
"created_at": "2026-01-01T00:00:00Z",
}
).encode()
def crash_at(expected):
def inject(point):
if point == expected:
raise SimulatedCrash(point)
return inject
def slow_builder(attachment_id, content, pipeline_version):
time.sleep(5)
def test_duplicate_and_ack_loss(content):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(1), job_body(),
fault_injector=crash_at("before_ack"),
)
raise AssertionError("fault injection did not crash before ack")
except SimulatedCrash:
pass
assert store.row["status"] == "ready"
first_write_count = len(store.objects.puts)
assert channel.acked == []
handle_delivery(store, channel, FakeMethod(2), job_body())
assert channel.acked == [2]
assert len(store.objects.puts) == first_write_count
assert store.row["status"] == "ready"
return {
"scenario": "duplicate_delivery_after_commit_before_ack",
"deliveries": 2,
"effective_object_writes": first_write_count,
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
}
def test_crash_before_and_after_side_effect(content, crash_point):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(3), job_body(),
fault_injector=crash_at(crash_point),
)
raise AssertionError(f"fault injection did not crash at {crash_point}")
except SimulatedCrash:
pass
assert channel.acked == []
store.recover_expired()
retry = job_body(attempt=2)
handle_delivery(store, channel, FakeMethod(4), retry)
assert store.row["status"] == "ready"
assert store.row["attempt"] == 2
assert channel.acked == [4]
assert store.objects.data["original/" + ATTACHMENT_ID] == content
return {
"scenario": "crash_" + crash_point,
"attempts": store.row["attempt"],
"terminal_status": store.row["status"],
}
def test_reordered_attempts(content):
store = FakeStore(content)
store.row["attempt"] = 2
channel = FakeChannel()
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
writes_after_current = len(store.objects.puts)
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
assert store.row["status"] == "ready"
assert len(store.objects.puts) == writes_after_current
assert channel.acked == [5, 6]
return {
"scenario": "out_of_order_attempt_delivery",
"current_attempt": store.row["attempt"],
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
}
def test_stale_pipeline_cannot_publish_newer_result(content):
store = FakeStore(content)
channel = FakeChannel()
replayed = []
def supersede_after_first_variant(point):
if point == "after_variant_put:thumbnail" and not replayed:
replayed.append(store.replay(2))
handle_delivery(
store, channel, FakeMethod(7), job_body(),
fault_injector=supersede_after_first_variant,
)
assert store.row["status"] == "queued"
assert store.row["pipeline_version"] == 2
assert channel.acked == [7]
current_job = replayed[0]
handle_delivery(
store,
channel,
FakeMethod(8),
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
)
assert store.row["status"] == "ready"
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
assert channel.acked == [7, 8]
return {
"scenario": "stale_pipeline_after_newer_version",
"current_pipeline_version": store.row["pipeline_version"],
"published_variant_versions": sorted(
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
),
}
class FakeTransaction:
def __init__(self, store):
self.store = store
def __enter__(self):
self.published = False
self.store.current_transaction = self
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None and self.published:
self.store.outbox_published = True
self.store.current_transaction = None
return False
class FakeConnection:
def __init__(self, store):
self.store = store
def transaction(self):
return FakeTransaction(self.store)
def execute(self, query, params):
assert "UPDATE attachment_outbox" in query
assert params == ("outbox-row",)
self.store.current_transaction.published = True
class FakePool:
def __init__(self, store):
self.store = store
def connection(self):
connection = FakeConnection(self.store)
class ConnectionContext:
def __enter__(self_inner):
return connection
def __exit__(self_inner, exc_type, exc_value, traceback):
return False
return ConnectionContext()
class FakeOutboxStore:
def __init__(self):
self.pool = FakePool(self)
self.outbox_published = False
self.current_transaction = None
def claim_outbox(self, connection):
if self.outbox_published:
return None
return {
"id": "outbox-row",
"job_id": JOB_ID,
"attachment_id": ATTACHMENT_ID,
"pipeline_version": 1,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
}
class AmbiguousConfirmChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
self.messages.append(message)
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
class FailBeforePublishChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
self.messages.append(message)
def test_committed_outbox_survives_publisher_crash():
store = FakeOutboxStore()
channel = FailBeforePublishChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher crash was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert channel.messages == []
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 1
return {
"scenario": "publisher_crash_before_publish",
"published_after_restart": len(channel.messages),
"outbox_recovered_after_restart": store.outbox_published,
}
def test_outbox_confirm_gap_republishes_same_job():
store = FakeOutboxStore()
channel = AmbiguousConfirmChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher confirm fault was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 2
first, second = channel.messages
assert first["body"] == second["body"]
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
return {
"scenario": "publish_confirm_lost_after_broker_accept",
"deliveries": len(channel.messages),
"same_message_id": True,
}
def test_image_processing_has_a_hard_timeout():
started = time.monotonic()
try:
_build_variants_bounded(
ATTACHMENT_ID,
b"bounded timeout test",
1,
timeout_seconds=1,
builder=slow_builder,
)
raise AssertionError("slow transform exceeded its time budget without termination")
except ProcessingTimeout:
pass
elapsed = time.monotonic() - started
assert elapsed < 4
return {
"scenario": "transform_timeout_terminated",
"configured_timeout_seconds": 1,
"observed_seconds": round(elapsed, 3),
}
def main():
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
results = [
test_duplicate_and_ack_loss(content),
test_crash_before_and_after_side_effect(content, "after_claim"),
test_crash_before_and_after_side_effect(content, "after_original_put"),
test_reordered_attempts(content),
test_stale_pipeline_cannot_publish_newer_result(content),
test_committed_outbox_survives_publisher_crash(),
test_outbox_confirm_gap_republishes_same_job(),
test_image_processing_has_a_hard_timeout(),
]
report = {
"result": "PASS",
"track": "2",
"fixture": "tests/fixtures/sample.png",
"scenarios": results,
"invariants": {
"confirmed_work_eventually_ready": True,
"duplicate_effects_create_no_second_variant_set": True,
"stale_attempt_writes_no_effects": True,
"stale_pipeline_cannot_publish_current_metadata": True,
"ambiguous_confirm_republishes_stable_job": True,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
},
}
evidence = ROOT / "evidence" / "pipeline-faults.json"
evidence.parent.mkdir(exist_ok=True)
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(report, sort_keys=True))
print(f"Evidence written to {evidence.relative_to(ROOT)}")
if __name__ == "__main__":
main()
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Verify explicit replay changes the deployed pipeline while preserving the original."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
MARK_FAILED_CODE = """
import sys
from src.store import Store
store = Store()
store.open()
try:
with store.pool.connection() as conn:
updated = conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', "
"variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'",
(sys.argv[1],),
)
if updated.rowcount != 1:
raise RuntimeError("test attachment was not ready for failed replay injection")
finally:
store.close()
"""
AUDIT_CODE = """
import json
import sys
from src.store import Store
store = Store()
store.open()
try:
with store.pool.connection() as conn:
rows = conn.execute(
"SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events "
"WHERE attachment_id = %s ORDER BY id",
(sys.argv[1],),
).fetchall()
print(json.dumps(rows, sort_keys=True))
finally:
store.close()
"""
def docker_exec(container_id, *args):
return subprocess.run(
["docker", "exec", container_id, *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def replay(container_id, attachment_id, version):
result = docker_exec(
container_id,
"python3",
"-m",
"src.replay_attachment",
"--attachment-id",
attachment_id,
"--pipeline-version",
str(version),
)
return json.loads(result.stdout)
def main():
args = arguments("Verify versioned attachment replay against the running Compose stack")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready_v1 = client.wait_ready(alice, attachment)
v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail")
v1_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch")
containers = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
).stdout.split()
check(bool(containers), "no running API container found")
replay_v2 = replay(containers[0], attachment["id"], 2)
check(replay_v2["status"] == "queued", "replay was not queued")
check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2")
check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id")
ready_v2 = client.wait_ready(alice, ready_v1)
v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail")
v2_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
original_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=alice,
)["bytes"]
check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding")
check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch")
check(original_bytes == original, "pipeline replay changed the original file")
docker_exec(
containers[0],
"python3",
"-c",
MARK_FAILED_CODE,
attachment["id"],
)
failed_replay = replay(containers[0], attachment["id"], 2)
check(failed_replay["status"] == "queued", "failed attachment replay was not queued")
check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version")
check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id")
ready_replayed = client.wait_ready(alice, ready_v2)
audit = json.loads(
docker_exec(
containers[0],
"python3",
"-c",
AUDIT_CODE,
attachment["id"],
).stdout
)
check(
[(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit]
== [(1, 2, "replay"), (2, 2, "replay")],
"pipeline replay audit trail is incomplete",
)
check(ready_replayed["status"] == "ready", "failed attachment did not recover")
print(
json.dumps(
{
"result": "PASS",
"previous_job_id": replay_v2["previous_job_id"],
"job_id": replay_v2["job_id"],
"pipeline_version": replay_v2["pipeline_version"],
"failed_replay_job_id": failed_replay["job_id"],
"audited_replays": len(audit),
"v1_thumbnail_sha256": v1_thumbnail["sha256"],
"v2_thumbnail_sha256": v2_thumbnail["sha256"],
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Deterministic tests of the reconciliation decision rules without deleting real objects."""
import os
import hashlib
import sys
import tempfile
from pathlib import Path
os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from datetime import datetime, timedelta, timezone
from src.reconcile import reconcile
class FakeObjects:
def __init__(self, items):
self.items = items
self.deleted = []
def objects(self):
return iter(self.items)
def delete(self, key):
self.deleted.append(key)
self.items = [(name, date) for name, date in self.items if name != key]
def put(self, key, content, content_type):
self.items.append((key, datetime.now(timezone.utc)))
class FakeStore:
def __init__(self):
now = datetime.now(timezone.utc)
self.metadata = {
"original/present": {"id": "present-id", "object_key": "original/present"},
"original/missing": {
"id": "missing-id",
"object_key": "original/missing",
"size_bytes": 4,
"sha256": hashlib.sha256(b"test").hexdigest(),
"content_type": "image/png",
},
}
self.objects = FakeObjects(
[
("original/present", now),
("original/stale-orphan", now - timedelta(hours=2)),
("original/new-orphan", now - timedelta(minutes=10)),
]
)
self.missing_incidents = set()
self.cleared_incidents = set()
def reconciliation_snapshot(self):
return self.metadata
def record_missing_object(self, attachment_id):
self.missing_incidents.add(attachment_id)
def clear_missing_object(self, attachment_id):
self.missing_incidents.discard(attachment_id)
self.cleared_incidents.add(attachment_id)
def main():
store = FakeStore()
dry_run = reconcile(store, grace_seconds=3600, apply_changes=False)
assert dry_run["stale_orphan_candidates"] == 1
assert dry_run["missing_objects"] == 1
assert dry_run["deleted_orphans"] == 0
assert store.objects.deleted == [] and store.missing_incidents == set()
applied = reconcile(store, grace_seconds=3600, apply_changes=True)
assert store.objects.deleted == ["original/stale-orphan"]
assert "original/new-orphan" not in store.objects.deleted
assert store.missing_incidents == {"missing-id"}
assert store.cleared_incidents == {"present-id"}
assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1
with tempfile.TemporaryDirectory() as backup:
restore_dir = Path(backup)
(restore_dir / "missing-id").write_bytes(b"test")
preview = reconcile(store, 3600, False, restore_dir)
assert preview["restorable_missing_objects"] == 1
assert store.missing_incidents == {"missing-id"}
restored = reconcile(store, 3600, True, restore_dir)
assert restored["restored_missing_objects"] == 1
assert store.missing_incidents == set()
repeated = reconcile(store, grace_seconds=3600, apply_changes=True)
assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0
assert store.objects.deleted == ["original/stale-orphan"]
assert store.missing_incidents == set()
print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period")
if __name__ == "__main__":
main()
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
import tempfile
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
STORE_CODE = """
import sys
from pathlib import Path
from src.store import Store
store = Store()
store.open()
try:
action = sys.argv[1]
if action == "key":
attachment_id = sys.argv[2]
matches = [
key for key, row in store.reconciliation_snapshot().items()
if str(row["id"]) == attachment_id and row["reconcile_type"] == "original"
]
if len(matches) != 1:
raise RuntimeError("attachment metadata lookup failed")
print(matches[0])
elif action == "put":
key, file_path = sys.argv[2:4]
content = Path(file_path).read_bytes()
store.objects.put(key, content, "image/png", len(content))
elif action == "delete":
store.objects.delete(sys.argv[2])
elif action == "exists":
print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no")
else:
raise RuntimeError("unsupported test operation")
finally:
store.close()
"""
PRIVATE_CHECK_CODE = """
import socket
import sys
import urllib.error
import urllib.request
try:
socket.getaddrinfo("s3-core", 8888)
except socket.gaierror:
print("PASS: SeaweedFS master/filer network is isolated")
else:
raise RuntimeError("API network can resolve the SeaweedFS core service")
for port in (8181, 9101):
try:
socket.create_connection(("s3", port), timeout=1).close()
except OSError:
continue
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
try:
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
except urllib.error.HTTPError as error:
if error.code == 403:
print("PASS: unauthenticated bucket access denied")
else:
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
else:
raise RuntimeError("private bucket allowed an unauthenticated GET")
"""
def docker_exec(container_id, *args):
return subprocess.run(
["docker", "exec", container_id, *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def reconciliation(container_id, *args):
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
return json.loads(result.stdout)
def object_exists(container_id, key):
return docker_exec(
container_id, "python3", "-c", STORE_CODE, "exists", key
).stdout.strip() == "yes"
def main():
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, content = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
containers = result.stdout.split()
check(len(containers) >= 2, "at least two running API replicas are required")
container_id = containers[0]
key = docker_exec(
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
).stdout.strip()
orphan_key = "original/" + str(uuid.uuid4())
with tempfile.TemporaryDirectory() as backup:
source = Path(backup) / "original.png"
source.write_bytes(content)
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
recovery_file = recovery_dir + "/" + attachment["id"]
orphan_created = False
orphan_removed = False
missing_deleted = False
missing_restored = False
try:
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
orphan_created = True
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
missing_deleted = True
preview = reconciliation(container_id)
check(
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
"dry-run missed injected inconsistencies",
)
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
immediate_apply = reconciliation(container_id, "--apply")
check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period")
docker_exec(container_id, "mkdir", "-p", recovery_dir)
subprocess.run(
["docker", "cp", str(source), container_id + ":" + recovery_file],
check=True,
)
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
missing_restored = True
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period")
print("Waiting for the 60-second orphan grace period.")
time.sleep(61)
cleanup_preview = reconciliation(container_id)
check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace")
cleaned = reconciliation(container_id, "--apply")
orphan_removed = not object_exists(container_id, orphan_key)
check(orphan_removed, "injected stale orphan was not removed")
final_scan = reconciliation(container_id, "--apply")
check(
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
"repeat scan is not clean",
)
finally:
if missing_deleted and not missing_restored:
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
if orphan_created and not orphan_removed:
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
check("PASS:" in private, "unauthenticated S3 access check failed")
downloaded = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)["bytes"]
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
print(
json.dumps(
{
"result": "PASS",
"dataset_objects_scanned": final_scan["objects_seen"],
"metadata_records_scanned": final_scan["metadata_seen"],
"scan_duration_ms": final_scan["duration_ms"],
"orphan_detected_and_removed": cleaned["deleted_orphans"],
"missing_detected_and_restored": restored["restored_missing_objects"],
"fresh_orphan_preserved_until_grace": True,
"message_attachment_download_checksum": "unchanged",
"private_bucket": "anonymous request denied",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Local transformation checks for the lab5 image worker."""
import hashlib
import io
import os
from pathlib import Path
import sys
os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret")
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from src.worker import build_variants
def main():
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
attachment_id = "00000000-0000-4000-8000-000000000001"
variants = build_variants(attachment_id, original, pipeline_version=1)
by_name = {item["name"]: item for item in variants}
assert set(by_name) == {"thumbnail", "crop", "optimized"}
assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160)
assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128)
assert by_name["optimized"]["content_type"] == "image/webp"
for item in variants:
content = item["content"]
assert len(content) == item["size_bytes"]
assert hashlib.sha256(content).hexdigest() == item["sha256"]
with Image.open(io.BytesIO(content)) as image:
assert image.size == (item["width"], item["height"])
newer = build_variants(attachment_id, original, pipeline_version=2)
assert all(f"/v2/" in item["object_key"] for item in newer)
assert all(f"/v1/" in item["object_key"] for item in variants)
assert by_name["thumbnail"]["sha256"] != {
item["name"]: item for item in newer
}["thumbnail"]["sha256"]
odd_ratio = io.BytesIO()
Image.new("RGB", (319, 200), "blue").save(odd_ratio, format="PNG")
odd_variants = {
item["name"]: item for item in build_variants(attachment_id, odd_ratio.getvalue())
}
assert (odd_variants["thumbnail"]["width"], odd_variants["thumbnail"]["height"]) == (256, 160)
small_image = io.BytesIO()
Image.new("RGB", (100, 50), "red").save(small_image, format="PNG")
small_variants = {
item["name"]: item for item in build_variants(attachment_id, small_image.getvalue())
}
assert (small_variants["thumbnail"]["width"], small_variants["thumbnail"]["height"]) == (100, 50)
assert (small_variants["optimized"]["width"], small_variants["optimized"]["height"]) == (100, 50)
print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums")
if __name__ == "__main__":
main()
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Verify queued uploads and message links survive an API/worker restart."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments("Verify queued attachment recovery after API and worker restart")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
compose("stop", "worker")
worker_stopped = True
try:
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not accepted while worker was stopped")
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message")
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
409,
actor=bob,
)
compose("restart", "api")
compose("start", "worker")
worker_stopped = False
client.call("GET", "/health/ready", schema="Health")
ready = client.wait_ready(bob, attachment)
original_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)["bytes"]
check(original_bytes == original, "recovered worker changed original bytes")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages",
actor=bob,
schema="MessagePage",
)["json"]
recovered_message = next(item for item in page["items"] if item["id"] == message["id"])
check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link")
print(
json.dumps(
{
"result": "PASS",
"initial_status": attachment["status"],
"final_status": ready["status"],
"message_attachment_link_survived": True,
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
},
sort_keys=True,
)
)
finally:
if worker_stopped:
compose("start", "worker")
if __name__ == "__main__":
run(main)
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
from concurrent.futures import ThreadPoolExecutor
import json
import os
import re
import subprocess
import threading
import time
import uuid
from io import BytesIO
from pathlib import Path
import sys
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
def usage_bytes(line):
match = USAGE_RE.search(line)
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
def api_containers():
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
ids = result.stdout.split()
check(len(ids) >= 2, "at least two running API containers are required")
return ids
def sample_once(ids):
output = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
check=True,
capture_output=True,
text=True,
).stdout.splitlines()
values = [usage_bytes(line) for line in output]
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
return max(values)
def main():
args = arguments("Bounded-memory ten-client upload experiment")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
width = height = 1400
content = BytesIO()
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
content, format="PNG", compress_level=0
)
payload = content.getvalue()
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
ids = api_containers()
baseline = sample_once(ids)
samples = []
stats_process = subprocess.Popen(
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
def sampler():
if stats_process.stdout is None:
return
for line in stats_process.stdout:
value = usage_bytes(line)
if value:
samples.append(value)
sampling_thread = threading.Thread(target=sampler, daemon=True)
sampling_thread.start()
count = 10
barrier = threading.Barrier(count)
def upload(index):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=15)
return worker.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
202,
actor=alice,
raw=payload,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
started = time.monotonic()
try:
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(upload, range(count)))
finally:
stopped_by_test = False
if stats_process.poll() is None:
stats_process.kill()
stopped_by_test = True
try:
stats_process.wait(timeout=5)
except subprocess.TimeoutExpired:
stats_process.kill()
stats_process.wait(timeout=5)
sampling_thread.join(timeout=5)
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
check(
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
"docker stats sampling process failed (exit "
+ str(stats_process.returncode)
+ "): "
+ stats_error.strip()[:300],
)
check(bool(samples), "docker stats produced no memory samples")
peak = max([baseline] + samples)
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
print(
json.dumps(
{
"uploads": count,
"concurrency": count,
"bytes_per_upload": len(payload),
"duration_seconds": round(time.monotonic() - started, 3),
"api_containers": len(ids),
"sample_count": len(samples) // len(ids),
"baseline_max_container_bytes": baseline,
"peak_max_container_bytes": peak,
"peak_delta_bytes": max(0, peak - baseline),
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)