forked from mtusi2026/mtusi2026lab5
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
abe73dfd2a | ||
|
|
ff99c9127a | ||
|
|
e85aba5296 | ||
|
|
f1dbe61eaf | ||
|
|
1f432c2e84 |
@@ -17,4 +17,10 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
student-tests
|
||||
tests
|
||||
research
|
||||
REPORT.md
|
||||
SCHEMA.md
|
||||
requirements-dev.txt
|
||||
.DS_Store
|
||||
|
||||
+5
-1
@@ -21,5 +21,9 @@ MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_TIMEOUT_SECONDS=20
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
PUBLIC_PORT=8085
|
||||
DATABASE_POOL_SIZE=8
|
||||
DATABASE_READ_AUTOCOMMIT=1
|
||||
|
||||
@@ -1,18 +1,16 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Горбанюк / БВТ2301.
|
||||
Целевая оценка: 5. Трек: 2 — идемпотентность versioned pipeline.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Реализация и воспроизводимая последовательность запуска: [REPORT.md](../REPORT.md).
|
||||
Исходники приложения — src/, собственные проверки — student-tests/, результаты — evidence/.
|
||||
Сохранено 84 собственных проверочных случаев; условия и результаты конкретных запусков приведены в отчёте.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Контракт, исходное задание и публичные тесты сохранены. Секреты создаются локально и не входят в Git.
|
||||
Ограничения и использованные первичные источники перечислены в REPORT.md.
|
||||
|
||||
- [x] Реализован обязательный минимум и заявленный трек.
|
||||
- [x] Приведены команды запуска, проверок и безопасной остановки.
|
||||
- [x] Сохранены результаты проверок и ограничения выводов.
|
||||
- [x] Credentials и приватные ключи находятся вне репозитория.
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: Application integration
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.14.7'
|
||||
- run: python -m pip install uv==0.12.13
|
||||
- run: uv sync --locked
|
||||
- run: make bootstrap lint check BASE_URL=http://localhost:8085
|
||||
- run: VCS_REF=$(git rev-parse HEAD) docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
|
||||
- run: PYTEST_ADDOPTS=--disable-warnings make unit
|
||||
- run: make test BASE_URL=http://localhost:8085
|
||||
- run: uv run --locked python scripts/verify_cluster.py --base-url http://localhost:8085 --output /tmp/cluster-ci.json
|
||||
- run: uv run python scripts/verify_pipeline.py --base-url http://localhost:8085 --output /tmp/pipeline-ci.json
|
||||
- if: always()
|
||||
run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml down
|
||||
@@ -0,0 +1 @@
|
||||
3.14.7
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
|
||||
WORKDIR /app
|
||||
RUN python -m pip install --no-cache-dir uv==0.12.13
|
||||
ENV UV_PYTHON_DOWNLOADS=never
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
|
||||
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY migrations /app/migrations
|
||||
ARG VCS_REF=uncommitted
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
APP_LOG_LEVEL=INFO
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
BASE_URL ?= http://127.0.0.1:8085
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -28,3 +28,21 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: bootstrap up down unit lint verify-cluster verify-pipeline
|
||||
bootstrap:
|
||||
$(PYTHON) scripts/bootstrap.py
|
||||
up:
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
|
||||
down:
|
||||
docker compose --env-file .env stop --timeout 7
|
||||
unit:
|
||||
uv run pytest --tb=short
|
||||
lint:
|
||||
uv run ruff check .
|
||||
uv run ruff format --check .
|
||||
verify-cluster:
|
||||
uv run python scripts/verify_cluster.py --base-url "$(BASE_URL)"
|
||||
verify-pipeline:
|
||||
uv run python scripts/verify_pipeline.py --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -1,37 +1,212 @@
|
||||
# Отчёт — лабораторная 5
|
||||
# Лабораторная 5 — очереди и обработка изображений
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
|
||||
## Запуск из чистого clone
|
||||
**Горбанюк, БВТ2301. Целевая оценка: 5, трек 2 — идемпотентность versioned pipeline с внедрением отказов.**
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Решение опубликовано в публичном форке исходного проекта; МР преподавателю открыт. Ветка `solution/bvt2301-gorbanyuk`, проверенная версия — тег `lab5-verified`. Название МР: **«ЛР 5 — БВТ2301 — Горбанюк — на 5»**. README задания, COURSE, CONTRIBUTING, контракты и публичные тесты сохранены.
|
||||
|
||||
## Архитектура и решения
|
||||
Публичный форк: <https://gitea.privcloud.tech/Wemiliam/mtusi2026lab5>. Ветка решения: `solution/bvt2301-gorbanyuk`, целевая ветка преподавателя — `main`. Ветка опубликована; МР преподавателю: <https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/10>.
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
## Запуск из самостоятельного clone
|
||||
|
||||
Нужны Git, Docker с Compose v2+, Python **3.14.7**, `uv 0.12.13`, доступ к реестрам и официальным Go-модулям при первом build. Проверено на ARM64, Docker 29.8.1 / Compose 5.5.1; Docker VM: 18 CPU, 7.75 GiB RAM. Выделите Docker около 6 GiB RAM и несколько GiB диска. Соседние лабораторные не нужны.
|
||||
|
||||
Команды выполняются из корня клонированного репозитория.
|
||||
|
||||
```sh
|
||||
uv sync --locked
|
||||
make bootstrap
|
||||
VCS_REF=$(git rev-parse HEAD) make up
|
||||
make check validate PYTHON=.venv/bin/python
|
||||
make test PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
API: `http://127.0.0.1:8085`, две реплики за HAProxy. У API нет собственных опубликованных портов. `bootstrap` создаёт случайные локальные секреты в игнорируемом `secrets/` и `.env`, сохраняя прежние значения. Каталог 0700; файлы читаются контейнерным UID через Compose secrets. Это локальные файлы, не централизованное зашифрованное хранилище секретов.
|
||||
|
||||
Одноразовый `migrate` применяет SQL под advisory lock, проверяя checksum ранее применённых миграций. `bucket-init` создаёт private buckets и IAM identities. API ждёт их успешного завершения. MinIO и mc собираются из официальных исходников, закреплённых на `RELEASE.2025-10-15T17-29-55Z` / `RELEASE.2025-08-13T08-35-41Z`, Go 1.26.7. Первый холодный build может занять несколько минут. Бинарник имеет label `DEVELOPMENT.GOGET`; фактическая версия исходников проверяется через Go build info, а не этот label. Это учебный pin, без заявления о текущей production-поддержке.
|
||||
|
||||
Для собственных интеграционных проверок включите локальные test ports и отдельный тестовый bucket:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
|
||||
make lint unit
|
||||
uv run python scripts/verify_pipeline.py --output /tmp/pipeline-new.json
|
||||
uv run python scripts/verify_cluster.py --output /tmp/cluster-new.json
|
||||
```
|
||||
|
||||
Override открывает **только loopback**: PG 55437, кэш 56384, брокер 56385, S3 59005. Он включает операторские single-use fault hooks и timeout job **5 s / lease 8 s**. Обычный Compose оставляет hooks выключенными, timeout **20 s / lease 23 s**. Не запускайте одновременно сценарии, меняющие состояние одного стенда. Выходной файл должен быть новым, предыдущие доказательства не перезаписываются.
|
||||
|
||||
Реальная проверка возраста и очистки:
|
||||
|
||||
```sh
|
||||
uv run python scripts/verify_cleanup.py seed --seed /tmp/gc-seed-new.json
|
||||
# Пока объекты стареют, выполните другие проверки.
|
||||
uv run python scripts/verify_cleanup.py verify --seed /tmp/gc-seed-new.json --output /tmp/gc-new.json
|
||||
```
|
||||
|
||||
Второй вызов дожидается >=301 s после terminal failed. Возраст S3 и часы не подменяются. Seed содержит только ID, без credentials. Скрипт проверяет orphan, failed retention, защиту active и опубликованных объектов; после проверки отложенная работа освобождается и доходит до ready. Быстрый unit-тест гонки с API моделирует возраст метаданных отдельно, не заменяя эту проверку.
|
||||
|
||||
```sh
|
||||
make down # Остановить, сохранив данные.
|
||||
docker compose --env-file .env up -d --wait # Возобновить.
|
||||
# Только намеренное удаление ВСЕХ учебных данных этого проекта:
|
||||
docker compose --env-file .env down --volumes
|
||||
```
|
||||
|
||||
Проект `mtusi-lab5` имеет собственные `postgres_data`, `s3_data`, `broker_data`. Кэш одноразовый: сессии хранятся в PostgreSQL. Секреты/volumes не входят в Git или bundle.
|
||||
|
||||
## Архитектура и транзакционные границы
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[Клиент] --> LB[HAProxy]
|
||||
LB --> A1[API 1]
|
||||
LB --> A2[API 2]
|
||||
A1 --> PG[(PostgreSQL)]
|
||||
A2 --> PG
|
||||
A1 --> S[(Приватный S3)]
|
||||
A2 --> S
|
||||
PG --> R[Outbox relay и watchdog]
|
||||
R --> B[(Valkey Streams AOF)]
|
||||
B --> W[Worker supervisor]
|
||||
W --> P[Ограниченные дочерние процессы]
|
||||
P --> S
|
||||
P --> PG
|
||||
```
|
||||
|
||||
Перенесены собственные FastAPI API, Argon2id, серверные Bearer-сессии, keyset pagination, членство в чате, PostgreSQL, кэш, две реплики и атомарная идемпотентность сообщений. Из ЛР4 сохранены приватность, ограничения файлов, оригинальные байты и связь с чатом. Объявленный переход: **201 ready → 202 queued**; queued/processing можно прикреплять к сообщению, failed нельзя.
|
||||
|
||||
БД — источник истины о состоянии, S3 — о байтах. Брокер — **отдельный от кэша** Valkey 8.1.3: persistent volume, AOF `appendfsync always`, RDB `60 1`, максимум 192 MiB данных, `noeviction`. Кэш — другой процесс без persistence, `allkeys-lru`. Переполнение/остановка брокера задерживает relay; PG outbox остаётся сохранённым.
|
||||
|
||||
1. API проверяет сессию/чат, имя, MIME, фактический поток <=10 MiB, время приёма <=15 s, целостность PNG/JPEG, <=40 млн пикселей. Pixel guard действует **до** полной распаковки. На API один upload slot, ожидание <=5 s, затем 429. Временный файл в tmpfs не является durable staging.
|
||||
2. БД создаёт upload reservation с lease 120 s. Проверенный оригинал записывается в persistent private S3 `staging/<UUID>`. Имя клиента не используется как ключ. PUT имеет ограниченные socket timeouts без автоматических повторов.
|
||||
3. **Одна PG-транзакция** блокирует/проверяет reservation, создаёт attachment=queued, job и outbox, отмечает handoff. После COMMIT API возвращает **202 со снимком queued**. Thumbnail ещё не обещан; worker может успеть завершить до первого GET.
|
||||
4. Relay выбирает due outbox через `FOR UPDATE SKIP LOCKED`, делает XADD, затем фиксирует published_at. Потеря ответа XADD/COMMIT вызывает повтор с тем же job_id. До шага 3 возможен непринятый orphan: общей транзакции S3+PG нет. Reservation и retention GC закрывают это окно без ложного 202.
|
||||
5. Worker получает COUNT=1 из consumer group. В короткой PG-транзакции захватывает job, увеличивает attempt, сохраняет fence/lease. Pillow работает вне SQL-транзакции, в **отдельном OS-процессе**. Supervisor держит не более двух child, убивает зависший по wall clock. Child RLIMIT_AS 768 MiB; worker container 1536 MiB / 2 CPU / tmpfs 64 MiB / pids 96.
|
||||
6. Worker сверяет исходный размер/SHA-256, повторно валидирует изображение, сохраняет original и три варианта в `objects/<attachment>/v<version>/<variant>`. После всех PUT атомарно публикует метаданные **при совпадении текущей версии, fence и действующего lease**.
|
||||
7. XACK означает устойчивый эффект. XACK+XDEL атомарны в Lua для единственной consumer group; pending не обрезаются через MAXLEN. XAUTOCLAIM восстанавливает неподтверждённые записи. При временной ошибке ACK выполняется после durable retry-intent, при failed — после terminal registry.
|
||||
|
||||
API readiness проверяет БД/миграции и S3; кэш, broker, relay и worker не входят в обязательный синхронный путь. При остановленном worker API способен устойчиво принять файл, но обработка задержится. Liveness проверяет только процесс. `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE`, S3 file credentials соответствуют `.env.example`; pool size=8, upload=10 MiB/40M px, concurrency=2, max attempts=5, staging retention=86400 s. Secret-поля Settings исключены из repr.
|
||||
|
||||
## Состояния, retry, версии и идемпотентность
|
||||
|
||||
`queued → processing → ready`; временная ошибка/истёкший lease возвращают queued с due time. Постоянная ошибка или пять неудач дают failed с безопасным error_code. **Attempt — управляемая попытка обработки, не число доставок брокера.** Envelope строго соответствует `contracts/jobs.schema.json`: schema_version/type/job_id/attachment_id/pipeline_version/attempt/created_at; trace context зарезервирован схемой. Bytes, credentials, filename, URL в stream не передаются. Недоверенный envelope проверяется; неверный помещается в служебный DLQ без копирования произвольных полей.
|
||||
|
||||
`attachment_jobs WHERE state='failed'` — durable failed registry. Audit фиксирует claim, публикации, отказы, replay, старые доставки, fence. Temporary storage/network ошибки повторяются; повреждение/неверный формат/пиксели/MemoryError постоянны. Supervisor timeout восстанавливает watchdog relay. Full jitter: равномерно от **0.25 s до min(30, 2^attempt)**, абсолютное время due сохраняется в БД. После пятой неудачи автоматических повторов нет.
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env exec worker python -m messenger.operator failed
|
||||
docker compose --env-file .env exec worker python -m messenger.operator replay <attachment-uuid>
|
||||
docker compose --env-file .env exec worker python -m messenger.operator new-version <attachment-uuid>
|
||||
docker compose --env-file .env run --rm reconciler
|
||||
docker compose --env-file .env run --rm reconciler python -m messenger.cleanup --apply
|
||||
```
|
||||
|
||||
CLI доступен только оператору локального стенда, не публичному HTTP клиенту. Replay разрешён только для failed с retained source: CLI проверяет наличие объекта, сохраняет job_id/версию, начинает новый ограниченный цикл попыток с audit. Ready replay отклоняется. New-version реализует **v1→v2**: optimized WebP quality 82→76; другие преобразования одинаковы. Source key фиксирован в job; для reprocess готового attachment берётся опубликованный original.
|
||||
|
||||
Ключ эффекта **(attachment_id, pipeline_version, variant)**. UNIQUE(attachment_id,pipeline_version) защищает logical job. Имена объектов детерминированы; повторные PUT занимают те же ключи, без второго набора вариантов. Старый процесс может дописать свой prefix v1, но не v2. PG fence/lease дополнительно запрещают устаревший COMMIT. Ready-дубль сверяет полноту и HEAD size/checksum, затем ACK без resize. Старые version/attempt не захватывают новую работу. **Менять алгоритм, сохраняя номер версии, нельзя:** это нарушит детерминированность.
|
||||
|
||||
## Изображения, приватность и очистка
|
||||
|
||||
| Вариант | Параметры |
|
||||
| --- | --- |
|
||||
| original | Побайтная копия, тот же MIME/размер/SHA-256, включая исходные EXIF |
|
||||
| thumbnail | JPEG quality 85/subsampling 0, LANCZOS, <=256×256, без увеличения |
|
||||
| crop | JPEG quality 85/subsampling 0, центральный квадрат 128×128, увеличение маленьких допустимо |
|
||||
| optimized | WebP quality 82(v1)/76(v2), method 4, <=1280×1280 без увеличения |
|
||||
|
||||
EXIF orientation применяется до преобразований; alpha компонуется на белый. Размеры вписывания округляются вниз, минимум 1 px: 320×200 → 256×160. Generated variants не содержат EXIF. WebP не обязан быть меньше каждого маленького исходника.
|
||||
|
||||
Metadata и каждый download требуют сессию/членство; чужое — 404, content queued/processing/failed — 409. Download идёт через API, без redirect, со своим MIME/размером, private/no-store/nosniff. Storage keys отсутствуют в публичных метаданных. S3 anonymous GET — 403. IAM разделён: API пишет staging/читает final; worker читает оба prefix/пишет final; cleanup читает/удаляет, не пишет; тестовая identity ограничена отдельным bucket. Root keys не доступны API/worker.
|
||||
|
||||
Cleanup по умолчанию dry-run, применение явное. Retention **86400 s**, hard minimum 300 s. Удаляются старые непринятые orphan, staging после опубликованного результата, остатки старых версий. Active queued/processing защищает объекты attachment. Текущие published originals/variants сохраняются даже без message-link. Failed source сохраняется >=retention после отказа, затем удаляется с source_expired/audit; replay потерянного источника не обещается.
|
||||
|
||||
Перед DELETE повторяется HEAD ETag/size/age; недоступность не считается 404. Коллекторы сериализованы advisory lock. Replay/new-version и GC согласуются row lock attachment; reservation защищает ещё не завершённый handoff API: expired/abandoned upload не сможет после удаления вернуть 202.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Команда/сценарий | Наблюдение и файл |
|
||||
| --- | --- | --- |
|
||||
| Контракт и исходные tests | `make check validate`, Git compare | PASS, неизменённый skeleton; `evidence/contract.txt`, `audit.json` |
|
||||
| 202, реальный thumbnail, сессии, ACL | `make test` | PASS; `evidence/smoke.txt` |
|
||||
| API, гонки, размеры/EXIF/alpha, replay, GC fence | `make unit` | 84 проверки; `evidence/unit.txt` |
|
||||
| Worker down → queued message → API restart → broker SIGKILL → ready | Первый сценарий `verify_pipeline.py` | Реальные CID/StartedAt, сохранённые Stream IDs; `evidence/pipeline.json` |
|
||||
| COMMIT/publish gap, publish до outbox COMMIT | Первые два сценария матрицы | Восстановление без ручной новой job; `pipeline.json` |
|
||||
| Crash до/после S3 и после result COMMIT до ACK | Три сценария worker | Attempts 2/2/1, один набор результатов; `pipeline.json` |
|
||||
| Transient/permanent, jitter, timeout, exhaustion, replay | Четыре сценария матрицы | Safe failed, lease recovery, audited CLI replay; `pipeline.json` |
|
||||
| Bounded concurrency/time/RAM, варианты | Supervisor + checksum/decode/cgroup | Три реальных варианта; `pipeline.json` |
|
||||
| Versioned pipeline, reorder, stale writer | v2 перед v1; 44 доставки и publication fence | Новая версия/байты не перезаписаны; `pipeline.json` |
|
||||
| Private S3, чтение другой API | Anonymous/IAM отрицательные checks; actual uploader stopped | 403 и download с другой реплики; `pipeline.json` |
|
||||
| Retention/cleanup | `verify_cleanup.py`, реальный возраст >=301 s | Active/ready сохранены, expired/orphan удалены; `cleanup.json` |
|
||||
| Failover/rolling/persistence | `verify_cluster.py`, публичные resilience hooks | Настоящие процессы/stores; `cluster.json`, `failover.txt`, `persistence.txt` |
|
||||
| Чистый clone | Новые secrets/volumes и отдельный Compose project | `clean-clone.json`, `clean-clone.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Исследование трека 2
|
||||
|
||||
## Эксперименты и отказы
|
||||
Гипотеза: at-least-once + детерминированные versioned keys + PG fence сохраняют подтверждённые работы и запрещают старому writer опубликоваться поверх нового. `verify_pipeline.py` воспроизводит 10 сценариев на настоящих PG/S3/Stream/processes. Hooks одноразовые, операторские, выключены в обычном Compose. Crash — `os._exit(87)` worker child/relay; hang убивает supervisor. After_effect расположен после PUT original, до полного результата; after_commit — после всех вариантов/COMMIT до ACK.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Raw JSON содержит время, attempts, audit/PID, object count, версии, CID/StartedAt, cgroup memory/CPU. Длительности включают остановки/ожидание redelivery, **не являются p95 обычного upload**. `memory.peak` — пик всей cgroup с дочерними процессами, не RSS одного процесса. Реальные варианты декодируются и проверяются по SHA-256. Инварианты проверяются assertions. Матрица перегрузки/concurrency не заявляется: выбран трек 2.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
### Измеренные результаты локального прогона
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
| Сценарий | Время, s | Attempts последнего цикла |
|
||||
| --- | ---: | ---: |
|
||||
| commit_publish_gap_api_restart_broker_sigkill | 12.72 | 1 |
|
||||
| publish_before_outbox_commit_duplicate | 14.33 | 1 |
|
||||
| worker_crash_before_effect | 13.89 | 2 |
|
||||
| worker_crash_after_effect | 13.86 | 2 |
|
||||
| worker_crash_after_commit | 11.78 | 1 |
|
||||
| transient_2 | 8.71 | 3 |
|
||||
| hang_1 | 13.66 | 2 |
|
||||
| permanent_1_failed_then_explicit_replay | 6.04 | 1 |
|
||||
| transient_5_failed_then_explicit_replay | 30.80 | 1 |
|
||||
| reorder_stale_v1_after_v2_and_publication_fence | 19.35 | 1 |
|
||||
|
||||
Матрица: 11 подтверждённых upload, потеряно 0, устаревших перезаписей 0; по 4 объекта текущей версии. У failed перед replay отдельно зафиксированы исходные attempts 1/5; число в таблице для них относится к успешному новому циклу. Пик worker cgroup 209.18 MiB; OOM kills=0. Измерение относится к небольшому fixture 320×200 и этому прогону.
|
||||
|
||||
Реальный GC: failed age 301.00 s; dry-run нашёл 14 кандидатов и ничего не удалил; apply удалил 14. Активный staging сохранён и затем обработан; 4 текущих published objects остались неизменными. Эти counts зависят от числа прошлых синтетических загрузок, критерии проверяются по конкретным seed ID.
|
||||
|
||||
|
||||
Отказы API под шестью потоками: реплика 1: 201 подтверждённых записей, первое успешное чтение/запись после начала действия через 1.91 s, ошибки 2.92%; реплика 2: 165 подтверждённых записей, первое успешное чтение/запись после начала действия через 1.86 s, ошибки 6.37%. Rolling replacement: 563 подтверждённых записей, все сохранены, ошибки 5.13%. Не обещается отсутствие транзиентных 5xx во время отказа; сообщения повторяются тем же Idempotency-Key. Публичный failover отдельно: 130 успешных записей, 2 неудачные попытки, первое восстановление 0.984 s. Persistence перезапустил реальные API/PG/S3/broker/cache/worker/relay/LB и сохранил действительную сессию. Эти показатели относятся к локальной синтетической проверке отказов, не к оценке пропускной способности очереди.
|
||||
|
||||
## Ответы на вопросы README
|
||||
|
||||
**Publisher confirm и consumer ack.** Confirm подтверждает приём от producer брокером; он не говорит, что worker сделал картинку. ACK подтверждает устойчивую обработку либо сохранённое решение о retry/failed. В RabbitMQ это разные механизмы. Здесь publisher boundary — ответ XADD при AOF appendfsync always; consumer boundary — XACK. Неопределённый исход publish приводит к повторной отправке; смерть после result COMMIT до XACK — к повторной доставке.
|
||||
|
||||
**Что сохранено при 202?** Все байты уже приняты и проверены, original находится в persistent staging, PG COMMIT содержит queued attachment, logical job и outbox-intent. Thumbnail ещё не обещан. Брокер может быть недоступен: relay отправит позже. Локальная память/tmpfile не входят в гарантию. До 202 неоднозначный отказ может оставить orphan; повтор upload создаёт новый ID, поскольку идемпотентность upload контрактом не задана.
|
||||
|
||||
**Почему нормальны повторы; где ключ?** Нельзя атомарно объединить PG COMMIT, S3 PUT и ACK разных систем. Всегда есть окно «эффект выполнен, подтверждение потеряно». Job_id стабилен при redelivery, UNIQUE защищает logical job, (attachment_id,pipeline_version,variant) — эффект, fence — право публикации. Это **не exactly-once delivery**: доставок/PUT может быть несколько; один текущий набор метаданных — свойство нашего идемпотентного эффекта в заявленной модели.
|
||||
|
||||
**Как отличить зависший worker от долгого?** По одному молчанию достоверно невозможно. Мы задаём бюджет и lease: до истечения захват действителен, после разрешено восстановление. Supervisor убивает child по timeout; relay после более длинного lease создаёт bounded retry. Легитимная долгая обработка тоже может быть прервана — цена конечного бюджета. Новая попытка получает новый fence; старая не может COMMIT. Количество процессов/prefetch ограничено; XAUTOCLAIM восстанавливает доставку, а PG lease определяет право на эффект.
|
||||
|
||||
Подходы: разделение HTTP/domain/storage; явные транзакции; transactional outbox; at-least-once consumer group; идемпотентный consumer; конечный автомат; fencing/leases; bounded retry/backoff/jitter; failed registry; versioned namespace; least privilege; fault injection и проверяемые инварианты. Очередь сама по себе этих свойств не даёт.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Один host/PG/S3/broker не переживут потерю всех дисков/VM. Backup, репликация stores и удалённый TLS не реализованы. HTTP — только loopback/синтетические данные. Остановка relay задерживает также lease recovery, сохраняя задания. Оператор наблюдает failed/backlog/disk и запускает cleanup; scheduler не добавлен. Восстановление утраченных published objects из backup не реализовано. Audit/jobs не архивируются автоматически. PG/S3 место конечно; admission по размеру backlog не исследовался.
|
||||
|
||||
Тесты — воспроизводимая проверка перечисленных окон, а не математическое доказательство всех interleavings. Проверены два worker slots и версии 1/2; большие картинки под максимальной конкуренцией отдельно не исследованы. Конечные RAM/time лимиты могут отклонить легитимный большой файл; OOM/timeout восстанавливаются по lease с конечным бюджетом попыток. SHA-256 проверяет целостность, не авторство. Новая версия алгоритма требует реализации/тестирования, не только изменения числа.
|
||||
|
||||
## Источники и библиотеки
|
||||
|
||||
Python 3.14.7; FastAPI 0.143.3/Pydantic 2.14.1/Uvicorn 0.54.0; psycopg 3.3.6, PostgreSQL 18.3; redis client 8.1.0, Valkey 8.1.3; boto3 1.43.109, Pillow 13.0.0; HAProxy 3.2.1. Python-зависимости закреплены `uv.lock`, images/source pins — Compose/Dockerfile/infra.
|
||||
|
||||
- Нормативные README, contracts/README.md, attachment-states.md, jobs.schema.json этого skeleton.
|
||||
- [Valkey XREADGROUP](https://valkey.io/commands/xreadgroup/), [XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [XACK](https://valkey.io/commands/xack/), [persistence](https://valkey.io/topics/persistence/).
|
||||
- [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acknowledgements](https://www.rabbitmq.com/docs/confirms) — теория границ, RabbitMQ в стенде не используется.
|
||||
- [PostgreSQL row locks](https://www.postgresql.org/docs/18/explicit-locking.html), [SKIP LOCKED](https://www.postgresql.org/docs/18/sql-select.html).
|
||||
- [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [file formats](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html).
|
||||
- [MinIO source pin](https://github.com/minio/minio/tree/RELEASE.2025-10-15T17-29-55Z), [mc source pin](https://github.com/minio/mc/tree/RELEASE.2025-08-13T08-35-41Z).
|
||||
|
||||
Заявленные результаты получены запуском сохранённых проверок; ограничения и операторские вмешательства описаны выше.
|
||||
|
||||
## Финальная проверка самостоятельности
|
||||
|
||||
Отдельный clone исходного commit `25e4ff9` с новыми secrets и пустыми volumes прошёл сборку, lint/format, offline/full contract checks, публичный smoke, все 84 собственных теста, полную матрицу из 10 сценариев и публичный persistence с той же сессией. Реальные StartedAt изменились у API, PG, S3, брокера, кэша, worker, relay и LB. SHA-256 каждого установленного Python-модуля совпал с исходниками на обеих API-репликах; OCI revision соответствует проверенному commit. Порты API не опубликованы. Raw второй матрицы — `evidence/clean-pipeline.json`.
|
||||
|
||||
После проверки временные контейнеры/volumes и clone удалены. Основной стенд остановлен, его данные сохранены. Финальный commit добавляет только отчёт/доказательства; исходники соответствуют проверенному `25e4ff9`. ЛР1–ЛР4 остались на прежних чистых версиях.
|
||||
|
||||
## Соответствие ревизий
|
||||
|
||||
После редактуры отчёта, шаблона и служебных метаданных сохранено соответствие прежних и текущих Git-ревизий: [revision-map.json](evidence/revision-map.json). Код приложения, миграции, зависимости, Dockerfile, конфигурация сервисов и исходные публичные проверки не изменены. Численные результаты и исторические image/source identifiers в evidence сохранены в исходном виде; их не следует трактовать как новый запуск измерений.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Only DB/cache test ports; API replicas always stay behind the LB.
|
||||
services:
|
||||
postgres:
|
||||
ports: ["127.0.0.1:${TEST_PG_PORT:-55437}:5432"]
|
||||
valkey:
|
||||
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56384}:6379"]
|
||||
s3:
|
||||
ports: ["127.0.0.1:${TEST_S3_PORT:-59005}:9000"]
|
||||
bucket-init:
|
||||
environment:
|
||||
CREATE_TEST_BUCKET: '1'
|
||||
broker:
|
||||
ports: ["127.0.0.1:${TEST_BROKER_PORT:-56385}:6379"]
|
||||
worker:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: '1'
|
||||
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
|
||||
relay:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: '1'
|
||||
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
|
||||
+255
@@ -0,0 +1,255 @@
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api: &api
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-uncommitted}
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_api_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_api_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab5
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
networks: [backend]
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql]
|
||||
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
|
||||
-c, shared_preload_libraries=pg_stat_statements]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: ${POSTGRES_CPU:-1}
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
networks: [backend]
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
|
||||
--maxmemory-policy, allkeys-lru]
|
||||
healthcheck:
|
||||
test: [CMD, valkey-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
networks: [backend]
|
||||
migrate:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.migrate]
|
||||
secrets: [database_url]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
valkey: {condition: service_healthy}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
lb:
|
||||
image: haproxy:3.2.1-alpine
|
||||
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
ports: ["127.0.0.1:${PUBLIC_PORT:-8085}:8080"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
networks: [backend]
|
||||
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command: [valkey-server, --appendonly, "yes", --appendfsync, always, --save, "60 1",
|
||||
--maxmemory, 192mb, --maxmemory-policy, noeviction]
|
||||
volumes: [broker_data:/data]
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
healthcheck:
|
||||
test: [CMD, valkey-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks: [backend]
|
||||
relay:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.relay]
|
||||
restart: unless-stopped
|
||||
healthcheck: {disable: true}
|
||||
secrets: [database_url, cache_url, broker_url]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
worker:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.worker]
|
||||
restart: unless-stopped
|
||||
cpus: 2
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
|
||||
healthcheck: {disable: true}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- broker_url
|
||||
- source: s3_worker_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_worker_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
|
||||
s3:
|
||||
image: mtusi-lab5-s3:2025-10-15
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
|
||||
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
|
||||
cap_drop: [ALL]
|
||||
cpus: 1
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live']
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks: [backend]
|
||||
bucket-init:
|
||||
image: mtusi-lab5-mc:2025-08-13
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: client}
|
||||
command: [/bootstrap-s3.sh]
|
||||
volumes:
|
||||
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
|
||||
- ./infra/api-policy.json:/policies/api-policy.json:ro
|
||||
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
|
||||
- ./infra/worker-policy.json:/policies/worker-policy.json:ro
|
||||
- ./infra/test-policy.json:/policies/test-policy.json:ro
|
||||
secrets: [s3_root_user, s3_root_password, s3_api_access_key_id, s3_api_secret_access_key,
|
||||
s3_reconciler_access_key_id, s3_reconciler_secret_access_key,
|
||||
s3_test_access_key_id, s3_test_secret_access_key,
|
||||
s3_worker_access_key_id, s3_worker_secret_access_key]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
reconciler:
|
||||
<<: *api
|
||||
profiles: [tools]
|
||||
command: [python, -m, messenger.cleanup]
|
||||
healthcheck: {disable: true}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_reconciler_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_reconciler_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
|
||||
networks:
|
||||
backend: {}
|
||||
volumes:
|
||||
broker_data:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_api_access_key_id:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
s3_test_access_key_id:
|
||||
file: ./secrets/s3_test_access_key_id
|
||||
s3_test_secret_access_key:
|
||||
file: ./secrets/s3_test_secret_access_key
|
||||
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
s3_worker_access_key_id:
|
||||
file: ./secrets/s3_worker_access_key_id
|
||||
s3_worker_secret_access_key:
|
||||
file: ./secrets/s3_worker_secret_access_key
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"working_files_checked": 104,
|
||||
"reachable_objects_checked": 128,
|
||||
"requested_names_absent": true,
|
||||
"actual_secret_values_absent": true,
|
||||
"public_requirements_unchanged": true,
|
||||
"upstream_push_disabled": true
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
{
|
||||
"source_commit": "25e4ff977ee932e507524d565e10c2cb9de8296e",
|
||||
"project": "mtusi-lab5-clean-1792259604",
|
||||
"image": "mtusi-messenger:lab5-verified-1792259604",
|
||||
"revision_label_matches": true,
|
||||
"fresh_secrets_and_volumes": true,
|
||||
"smoke_passed": true,
|
||||
"own_cases_passed": 84,
|
||||
"full_fault_matrix_passed": true,
|
||||
"same_session_persistence_passed": true,
|
||||
"all_services_restarted": [
|
||||
"api",
|
||||
"postgres",
|
||||
"s3",
|
||||
"broker",
|
||||
"valkey",
|
||||
"worker",
|
||||
"relay",
|
||||
"lb"
|
||||
],
|
||||
"before": {
|
||||
"api": [
|
||||
{
|
||||
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
|
||||
"started_at": "2026-10-17T17:54:22.360219637Z",
|
||||
"pid": 86502,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
},
|
||||
{
|
||||
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
|
||||
"started_at": "2026-10-17T17:56:41.112984132Z",
|
||||
"pid": 1849,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
}
|
||||
],
|
||||
"postgres": [
|
||||
{
|
||||
"id": "2b1547e1e079a74d5b2c11917b8f865d45449daf67232cf93736c24d3c72d452",
|
||||
"started_at": "2026-10-17T17:53:35.127862462Z",
|
||||
"pid": 81760,
|
||||
"ports": {
|
||||
"5432/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "55437"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db"
|
||||
}
|
||||
],
|
||||
"s3": [
|
||||
{
|
||||
"id": "eed453c4decc6ffcf3e9691ad06f3d5beacbf32ac311a8d3df812f7fc2a23886",
|
||||
"started_at": "2026-10-17T17:53:34.938232295Z",
|
||||
"pid": 81607,
|
||||
"ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59005"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:96efc3b1fee5c9f362cd39d5c2287f903c67c66cfd8b1ee7a112242978b2822b"
|
||||
}
|
||||
],
|
||||
"broker": [
|
||||
{
|
||||
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
|
||||
"started_at": "2026-10-17T17:54:27.551506417Z",
|
||||
"pid": 87340,
|
||||
"ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56385"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
|
||||
}
|
||||
],
|
||||
"valkey": [
|
||||
{
|
||||
"id": "d9bc56c3384fff2b3db1b898d8a09b9aaf077b238a01816c9cb7fc650de012ff",
|
||||
"started_at": "2026-10-17T17:53:35.027799337Z",
|
||||
"pid": 81685,
|
||||
"ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56384"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
|
||||
}
|
||||
],
|
||||
"worker": [
|
||||
{
|
||||
"id": "fd613952c9ad7d7b71a779e38eb082eb78d449f2fecbac578ecaa2ab1786d74d",
|
||||
"started_at": "2026-10-17T17:56:19.573687511Z",
|
||||
"pid": 99682,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
}
|
||||
],
|
||||
"relay": [
|
||||
{
|
||||
"id": "edbf313e6e97397f63432afa4c9799ee9e510c6621b31a56dd6d6859c4b384f8",
|
||||
"started_at": "2026-10-17T17:56:18.466104844Z",
|
||||
"pid": 99383,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
}
|
||||
],
|
||||
"lb": [
|
||||
{
|
||||
"id": "3e1e4cef40a73f29ebd16dc44465cc1e3151b6c269b8f98f297ff5f3b516bea9",
|
||||
"started_at": "2026-10-17T17:53:45.990409384Z",
|
||||
"pid": 83113,
|
||||
"ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "28085"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9"
|
||||
}
|
||||
]
|
||||
},
|
||||
"after": {
|
||||
"api": [
|
||||
{
|
||||
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
|
||||
"started_at": "2026-10-17T17:56:49.518161302Z",
|
||||
"pid": 3995,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
},
|
||||
{
|
||||
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
|
||||
"started_at": "2026-10-17T17:56:49.497895719Z",
|
||||
"pid": 3914,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
}
|
||||
],
|
||||
"postgres": [
|
||||
{
|
||||
"id": "2b1547e1e079a74d5b2c11917b8f865d45449daf67232cf93736c24d3c72d452",
|
||||
"started_at": "2026-10-17T17:56:49.373826427Z",
|
||||
"pid": 3766,
|
||||
"ports": {
|
||||
"5432/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "55437"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db"
|
||||
}
|
||||
],
|
||||
"s3": [
|
||||
{
|
||||
"id": "eed453c4decc6ffcf3e9691ad06f3d5beacbf32ac311a8d3df812f7fc2a23886",
|
||||
"started_at": "2026-10-17T17:56:49.225153469Z",
|
||||
"pid": 3581,
|
||||
"ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59005"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:96efc3b1fee5c9f362cd39d5c2287f903c67c66cfd8b1ee7a112242978b2822b"
|
||||
}
|
||||
],
|
||||
"broker": [
|
||||
{
|
||||
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
|
||||
"started_at": "2026-10-17T17:56:49.426393761Z",
|
||||
"pid": 3837,
|
||||
"ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56385"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
|
||||
}
|
||||
],
|
||||
"valkey": [
|
||||
{
|
||||
"id": "d9bc56c3384fff2b3db1b898d8a09b9aaf077b238a01816c9cb7fc650de012ff",
|
||||
"started_at": "2026-10-17T17:56:49.328862136Z",
|
||||
"pid": 3654,
|
||||
"ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56384"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
|
||||
}
|
||||
],
|
||||
"worker": [
|
||||
{
|
||||
"id": "fd613952c9ad7d7b71a779e38eb082eb78d449f2fecbac578ecaa2ab1786d74d",
|
||||
"started_at": "2026-10-17T17:56:49.806513386Z",
|
||||
"pid": 4189,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
}
|
||||
],
|
||||
"relay": [
|
||||
{
|
||||
"id": "edbf313e6e97397f63432afa4c9799ee9e510c6621b31a56dd6d6859c4b384f8",
|
||||
"started_at": "2026-10-17T17:56:49.443181386Z",
|
||||
"pid": 3840,
|
||||
"ports": {},
|
||||
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
|
||||
}
|
||||
],
|
||||
"lb": [
|
||||
{
|
||||
"id": "3e1e4cef40a73f29ebd16dc44465cc1e3151b6c269b8f98f297ff5f3b516bea9",
|
||||
"started_at": "2026-10-17T17:56:49.161912927Z",
|
||||
"pid": 3501,
|
||||
"ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "28085"
|
||||
}
|
||||
]
|
||||
},
|
||||
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9"
|
||||
}
|
||||
]
|
||||
},
|
||||
"module_sha256": {
|
||||
"worker.py": "37168828d2d01ec13bd1a108785b1f5afeac007a3fa7790b6172599edfd2eb34",
|
||||
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
|
||||
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
|
||||
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298",
|
||||
"operator.py": "04b22eb35647e721b0460f251db6cd3a4f07826d141814e8142ca1b683a15755",
|
||||
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
|
||||
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
|
||||
"cleanup.py": "4a5f08994bbe7defa997bd154baf2d973de68ca4aab934683a316b514a53430d",
|
||||
"attachments.py": "8e6a0924809332b2020e3abc2c0b89d6dec6a83d287c264368eee3048a31f2e7",
|
||||
"schemas.py": "5cca4d2ed64401fb2b9f530f3646f3bdfb979b82f050bd8d3126dfa6b5f62af7",
|
||||
"relay.py": "7c1d565187d0f42ac08f17bfaf6bb4df770e42094845c438371513ce90bcc60d",
|
||||
"pipeline.py": "37dfe1b31cdefdac10ffe6808b125d456623681edb886787898f0b8fe29501a7",
|
||||
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
|
||||
"images.py": "a01914d47ed69cee1605a3f2f9dffb9d90fa7850b8d2be5c37694af6ef8861cb",
|
||||
"settings.py": "9ae56366d9156db93c888bf1451ac4c55d9cfa5f2a2b70a47dd9170ecf4af149",
|
||||
"app.py": "2774f40d323a9d28839c47c7c0e7703d76b69d58af0ebf2222fb33b6df15435b",
|
||||
"broker.py": "dfa9bc04bf84cb9f0277ce3a86b5ac8062ec96e69278070f84b581977ae20240",
|
||||
"repository.py": "91ec54c8032ed283e39604ca0405a9752339096aa559498f2e9c7d099c8e47ba",
|
||||
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
|
||||
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
|
||||
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f",
|
||||
"faults.py": "f3bd0446a7024074e66df1aa07245f85824a94a24e9dafb4039a352e5bd42ec9"
|
||||
},
|
||||
"both_api_modules_match": true,
|
||||
"api_ports_unpublished": true,
|
||||
"private_values_absent_from_logs": true,
|
||||
"checkout_clean_after_tests": true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"retention_seconds": 300,
|
||||
"failed_real_age_seconds": 301.00391,
|
||||
"seed": {
|
||||
"created_at": "2026-10-17T17:45:40.213510+00:00",
|
||||
"active_id": "a406c829-1654-4e43-aab0-dd6f25df22ad",
|
||||
"failed_id": "d93b3943-9788-4179-81f7-a7e9a4134ca1",
|
||||
"orphan_id": "35f731d5-ab06-4c68-9a58-4b808ad4c1be"
|
||||
},
|
||||
"dry_run": {
|
||||
"scanned": 118,
|
||||
"eligible": 14,
|
||||
"deleted": 0,
|
||||
"protected": 49
|
||||
},
|
||||
"apply": {
|
||||
"scanned": 118,
|
||||
"eligible": 14,
|
||||
"deleted": 14,
|
||||
"protected": 49
|
||||
},
|
||||
"active_protected_then_ready": true,
|
||||
"orphan_deleted": true,
|
||||
"failed_source_expired": true,
|
||||
"published_objects_unchanged": 4
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
{
|
||||
"initial_instances": [
|
||||
{
|
||||
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
|
||||
"started_at": "2026-10-17T17:48:24.316494638Z",
|
||||
"pid": 61117,
|
||||
"running": true,
|
||||
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "d06f1ce7a2ad48fb85a6e4c959495a6b"
|
||||
},
|
||||
{
|
||||
"id": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
|
||||
"started_at": "2026-10-17T17:45:53.909326888Z",
|
||||
"pid": 45122,
|
||||
"running": true,
|
||||
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "4d81bcd18cd841d58fb480df271130ec"
|
||||
}
|
||||
],
|
||||
"failures": [
|
||||
{
|
||||
"duration_seconds": 5.945994583000356,
|
||||
"workers": 6,
|
||||
"requests": 2092,
|
||||
"successes": 2031,
|
||||
"errors": 61,
|
||||
"successful_rps": 341.5744786930423,
|
||||
"error_fraction": 0.029158699808795412,
|
||||
"first_success_after_action_seconds": 1.9096203330009303,
|
||||
"action_seconds": 1.902683000000252,
|
||||
"p50_ms": 11.905290999493445,
|
||||
"p95_ms": 57.65445899851329,
|
||||
"p99_ms": 78.713124999922,
|
||||
"instances": [
|
||||
"4d81bcd18cd841d58fb480df271130ec",
|
||||
"d06f1ce7a2ad48fb85a6e4c959495a6b"
|
||||
],
|
||||
"tail_instances": [
|
||||
"4d81bcd18cd841d58fb480df271130ec"
|
||||
],
|
||||
"confirmed_writes": 201,
|
||||
"stopped": {
|
||||
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
|
||||
"started_at": "2026-10-17T17:48:24.316494638Z",
|
||||
"pid": 61117,
|
||||
"running": true,
|
||||
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "d06f1ce7a2ad48fb85a6e4c959495a6b"
|
||||
}
|
||||
},
|
||||
{
|
||||
"duration_seconds": 5.482812500000364,
|
||||
"workers": 6,
|
||||
"requests": 1806,
|
||||
"successes": 1691,
|
||||
"errors": 115,
|
||||
"successful_rps": 308.4183528070471,
|
||||
"error_fraction": 0.0636766334440753,
|
||||
"first_success_after_action_seconds": 1.8648894590005511,
|
||||
"action_seconds": 1.440443999999843,
|
||||
"p50_ms": 13.768250000794069,
|
||||
"p95_ms": 50.30191599871614,
|
||||
"p99_ms": 71.77345800118928,
|
||||
"instances": [
|
||||
"4d81bcd18cd841d58fb480df271130ec",
|
||||
"99a8d6532148477f8630cabe3f0c88d8"
|
||||
],
|
||||
"tail_instances": [
|
||||
"99a8d6532148477f8630cabe3f0c88d8"
|
||||
],
|
||||
"confirmed_writes": 165,
|
||||
"stopped": {
|
||||
"id": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
|
||||
"started_at": "2026-10-17T17:45:53.909326888Z",
|
||||
"pid": 45122,
|
||||
"running": true,
|
||||
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "4d81bcd18cd841d58fb480df271130ec"
|
||||
}
|
||||
}
|
||||
],
|
||||
"concurrent_replays": 30,
|
||||
"concurrent_effects": 1,
|
||||
"rolling": {
|
||||
"duration_seconds": 13.661339999998745,
|
||||
"workers": 6,
|
||||
"requests": 5950,
|
||||
"successes": 5645,
|
||||
"errors": 305,
|
||||
"successful_rps": 413.2098315392574,
|
||||
"error_fraction": 0.05126050420168067,
|
||||
"first_success_after_action_seconds": 9.612477874999968,
|
||||
"action_seconds": 9.61113162500078,
|
||||
"p50_ms": 9.479749998718034,
|
||||
"p95_ms": 53.65141699985543,
|
||||
"p99_ms": 78.18379200034542,
|
||||
"instances": [
|
||||
"3e551e0479ef40d4ab399c2c97515ce0",
|
||||
"94c58fccf2e045e196c07dd588a92c1a",
|
||||
"98396ce0e5b4472c970589a37b35f629",
|
||||
"99a8d6532148477f8630cabe3f0c88d8"
|
||||
],
|
||||
"tail_instances": [
|
||||
"3e551e0479ef40d4ab399c2c97515ce0",
|
||||
"94c58fccf2e045e196c07dd588a92c1a"
|
||||
],
|
||||
"confirmed_writes": 563,
|
||||
"replacements": [
|
||||
{
|
||||
"old_container": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
|
||||
"old_instance": "99a8d6532148477f8630cabe3f0c88d8",
|
||||
"live_instances": [
|
||||
"94c58fccf2e045e196c07dd588a92c1a",
|
||||
"98396ce0e5b4472c970589a37b35f629"
|
||||
]
|
||||
},
|
||||
{
|
||||
"old_container": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
|
||||
"old_instance": "98396ce0e5b4472c970589a37b35f629",
|
||||
"live_instances": [
|
||||
"3e551e0479ef40d4ab399c2c97515ce0",
|
||||
"94c58fccf2e045e196c07dd588a92c1a"
|
||||
]
|
||||
}
|
||||
],
|
||||
"confirmed_messages_preserved": true
|
||||
},
|
||||
"persistence": {
|
||||
"before": {
|
||||
"postgres": {
|
||||
"id": "45876dc9476e2b47667d1bde173e786cb20aba45a179caca86a374f732cba70b",
|
||||
"started_at": "2026-10-17T17:38:39.558313298Z",
|
||||
"pid": 7427,
|
||||
"running": true,
|
||||
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"5432/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "55437"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"valkey": {
|
||||
"id": "4a11709684b5cdd4f6e4488ae0e11ef42085b4fdeace1e3c7cff88b05d0e9d1a",
|
||||
"started_at": "2026-10-17T17:37:37.056307755Z",
|
||||
"pid": 1093,
|
||||
"running": true,
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56384"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"s3": {
|
||||
"id": "9ab4050036931ffa30695ee3e29e57a0d767660324585a39fc21e8a8456e8ffc",
|
||||
"started_at": "2026-10-17T17:37:37.243124088Z",
|
||||
"pid": 1260,
|
||||
"running": true,
|
||||
"image": "sha256:94aff3c743f63f1a00f1b8d0ea1553fd31f21ffe622a6e70eecae1026546ec3c",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59005"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"broker": {
|
||||
"id": "e21054f56ca9160a2587b123cb21624e2685ebc7e252df354add56c704032f7b",
|
||||
"started_at": "2026-10-17T17:46:01.597192669Z",
|
||||
"pid": 46402,
|
||||
"running": true,
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56385"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"lb": {
|
||||
"id": "12688085914dbd56dce6dec01d5161bf371b45cec95a99b5510dc3782715f91a",
|
||||
"started_at": "2026-10-17T17:38:48.530733677Z",
|
||||
"pid": 8643,
|
||||
"running": true,
|
||||
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8085"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"postgres": {
|
||||
"id": "45876dc9476e2b47667d1bde173e786cb20aba45a179caca86a374f732cba70b",
|
||||
"started_at": "2026-10-17T17:51:16.499957968Z",
|
||||
"pid": 75615,
|
||||
"running": true,
|
||||
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"5432/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "55437"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"valkey": {
|
||||
"id": "4a11709684b5cdd4f6e4488ae0e11ef42085b4fdeace1e3c7cff88b05d0e9d1a",
|
||||
"started_at": "2026-10-17T17:51:16.385997843Z",
|
||||
"pid": 75510,
|
||||
"running": true,
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56384"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"s3": {
|
||||
"id": "9ab4050036931ffa30695ee3e29e57a0d767660324585a39fc21e8a8456e8ffc",
|
||||
"started_at": "2026-10-17T17:51:16.228970093Z",
|
||||
"pid": 75341,
|
||||
"running": true,
|
||||
"image": "sha256:94aff3c743f63f1a00f1b8d0ea1553fd31f21ffe622a6e70eecae1026546ec3c",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59005"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"broker": {
|
||||
"id": "e21054f56ca9160a2587b123cb21624e2685ebc7e252df354add56c704032f7b",
|
||||
"started_at": "2026-10-17T17:51:16.531699968Z",
|
||||
"pid": 75682,
|
||||
"running": true,
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56385"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"lb": {
|
||||
"id": "12688085914dbd56dce6dec01d5161bf371b45cec95a99b5510dc3782715f91a",
|
||||
"started_at": "2026-10-17T17:51:16.329691384Z",
|
||||
"pid": 75442,
|
||||
"running": true,
|
||||
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8085"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"same_token_public_test": true
|
||||
},
|
||||
"final_instances": [
|
||||
{
|
||||
"id": "2dfd89da126d334f744a65b1db2e0a7da97fe437f5480bf8ad8730657deeb7d2",
|
||||
"started_at": "2026-10-17T17:51:16.725000593Z",
|
||||
"pid": 75960,
|
||||
"running": true,
|
||||
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "aa801fbb4c3547bbb6d90db5fd31db59"
|
||||
},
|
||||
{
|
||||
"id": "28997d75263b8c92dcd8afe813478c473ad1eef479ed996dce86916ad245c75e",
|
||||
"started_at": "2026-10-17T17:51:16.571504468Z",
|
||||
"pid": 75797,
|
||||
"running": true,
|
||||
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "b22ea2f25f6546f88ddac407dd792c07"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
.venv/bin/python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8085"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 2, "first_recovery_seconds": 0.984}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8085"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"scope": "Редактура документации и метаданных рисунка",
|
||||
"runtime_sources_unchanged": true,
|
||||
"migrations_dependencies_and_infrastructure_unchanged": true,
|
||||
"instructor_contract_and_public_tests_unchanged": true,
|
||||
"measurement_values_unchanged": true,
|
||||
"original_head": "dc11d065fb13d155c19612b95d9b42704da60f9f",
|
||||
"rewritten_history_head": "f1dbe61eafc3c7254d2351a160ed63099cb38678",
|
||||
"commit_map": {
|
||||
"25e4ff977ee932e507524d565e10c2cb9de8296e": "1f432c2e848c89ca538c02aa5d7c41332d62b0c7",
|
||||
"276dbdad8a34da4c5b57fbf1879ca169b581d75b": "276dbdad8a34da4c5b57fbf1879ca169b581d75b",
|
||||
"dc11d065fb13d155c19612b95d9b42704da60f9f": "f1dbe61eafc3c7254d2351a160ed63099cb38678"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:8085"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,13 @@
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.7, pytest-9.1.1, pluggy-1.7.0
|
||||
rootdir: <repository>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: anyio-4.15.1
|
||||
collected 84 items
|
||||
|
||||
student-tests/test_api.py .................................... [ 42%]
|
||||
student-tests/test_cluster.py ............ [ 57%]
|
||||
student-tests/test_pipeline.py .................................... [100%]
|
||||
|
||||
======================== 84 passed, 1 warning in 18.40s ========================
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Never print mc arguments/errors: they can include access key identifiers.
|
||||
exec 3>&1
|
||||
exec > /tmp/bootstrap.log 2>&1
|
||||
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
|
||||
bucket="$S3_BUCKET"
|
||||
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
|
||||
mc --config-dir /tmp/mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
|
||||
for role in api worker reconciler test; do
|
||||
target_bucket="$bucket"
|
||||
policy_source=/policies/api-policy.json
|
||||
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
|
||||
if [ "$role" = worker ]; then policy_source=/policies/worker-policy.json; fi
|
||||
if [ "$role" = test ]; then
|
||||
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
|
||||
policy_source=/policies/test-policy.json
|
||||
target_bucket="$bucket-tests"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
|
||||
fi
|
||||
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
|
||||
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
|
||||
key="$(cat /run/secrets/s3_${role}_access_key_id)"
|
||||
mc --config-dir /tmp/mc admin user add local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
|
||||
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
|
||||
done
|
||||
trap - EXIT
|
||||
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
|
||||
@@ -0,0 +1,44 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2048
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
|
||||
timeout connect 500ms
|
||||
timeout check 500ms
|
||||
timeout client 20s
|
||||
timeout server 30s
|
||||
timeout http-request 3s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 0s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
|
||||
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
|
||||
|
||||
listen metrics
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 1s
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Official security-fixed source release; no dependency on retired image tags.
|
||||
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
|
||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
|
||||
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
|
||||
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
|
||||
COPY --from=builder /go/bin/minio /usr/local/bin/minio
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
|
||||
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
|
||||
COPY --from=builder /go/bin/mc /usr/local/bin/mc
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/bin/sh"]
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id,user_id),
|
||||
UNIQUE (chat_id,position)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE message_requests (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
idempotency_key varchar(128) COLLATE "C" NOT NULL
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
|
||||
response_message_id uuid NOT NULL REFERENCES messages(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (id,chat_id),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
ALTER TABLE messages ADD UNIQUE (id,chat_id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position),
|
||||
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
|
||||
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,46 @@
|
||||
-- Preserve earlier migration checksums; transition lab4 storage to versioned jobs.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_storage_state_check;
|
||||
ALTER TABLE attachments ADD CHECK (storage_state IN ('queued','processing','ready','failed')) NOT VALID;
|
||||
UPDATE attachments SET storage_state='queued' WHERE storage_state IN ('pending','missing');
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(variants)='array');
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
source_key text NOT NULL,
|
||||
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
fence uuid,
|
||||
lease_until timestamptz,
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
error_code text,
|
||||
source_expired boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until,next_attempt_at);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
stream_id text
|
||||
);
|
||||
CREATE INDEX outbox_due ON job_outbox(due_at) WHERE published_at IS NULL;
|
||||
CREATE TABLE pipeline_audit (
|
||||
sequence bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
event text NOT NULL,
|
||||
details jsonb NOT NULL DEFAULT '{}',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Only an operator with DB access can configure faults; production ignores this table.
|
||||
CREATE TABLE pipeline_faults (
|
||||
job_id uuid REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
phase text NOT NULL,
|
||||
action text NOT NULL CHECK (action IN ('crash','hang','transient','permanent')),
|
||||
remaining integer NOT NULL DEFAULT 1 CHECK (remaining >= 0),
|
||||
PRIMARY KEY(job_id,phase)
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- A paused API must not acknowledge an object that GC has already retired.
|
||||
CREATE TABLE upload_reservations (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL DEFAULT 'uploading' CHECK (state IN ('uploading','accepted','abandoned')),
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,46 @@
|
||||
[project]
|
||||
name = "mtusi-messenger"
|
||||
version = "0.5.0"
|
||||
description = "MTUSI lab 5: durable versioned image pipeline"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"argon2-cffi==25.1.0",
|
||||
"boto3==1.43.109",
|
||||
"fastapi==0.143.3",
|
||||
"pillow==13.0.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"pydantic==2.14.1",
|
||||
"redis==8.1.0",
|
||||
"uvicorn==0.54.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling==1.27.0"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/messenger"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"httpx==0.28.1",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
"pytest==9.1.1",
|
||||
"ruff==0.17.1",
|
||||
]
|
||||
research = ["matplotlib==3.11.2"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra --tb=short"
|
||||
|
||||
[tool.ruff]
|
||||
extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"]
|
||||
target-version = "py314"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once, without printing their values."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = directory / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_urlsafe(32) + "\n")
|
||||
password = password_file.read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
|
||||
"cache_url": "redis://valkey:6379/0",
|
||||
"broker_url": "redis://broker:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
# Compose bind-mount secrets must be readable by the non-root container process.
|
||||
# The parent directory remains 0700 on the host.
|
||||
for role in ("root", "api", "worker", "reconciler", "test"):
|
||||
names = (
|
||||
("s3_root_user", "s3_root_password")
|
||||
if role == "root"
|
||||
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
|
||||
)
|
||||
for index, name in enumerate(names):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(
|
||||
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
|
||||
)
|
||||
for name in (
|
||||
"postgres_password",
|
||||
"database_url",
|
||||
"cache_url",
|
||||
"broker_url",
|
||||
*[p.name for p in directory.glob("s3_*")],
|
||||
):
|
||||
os.chmod(directory / name, 0o644)
|
||||
env = root / ".env"
|
||||
if not env.exists():
|
||||
env.write_text((root / ".env.example").read_text())
|
||||
print("Local secret files and environment are ready; existing values preserved.")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="http://127.0.0.1:8085", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Local test operator configuration. Credentials stay in ignored files/in memory."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.settings import Settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def settings(role="worker"):
|
||||
prefix = "s3_root" if role == "root" else "s3_" + role
|
||||
access, secret = (
|
||||
("user", "password") if role == "root" else ("access_key_id", "secret_access_key")
|
||||
)
|
||||
return Settings(
|
||||
make_conninfo(
|
||||
host="127.0.0.1",
|
||||
port=int(os.getenv("TEST_PG_PORT", "55437")),
|
||||
user="messenger",
|
||||
dbname="messenger",
|
||||
password=(ROOT / "secrets/postgres_password").read_text().strip(),
|
||||
),
|
||||
"redis://127.0.0.1:" + os.getenv("TEST_CACHE_PORT", "56384") + "/0",
|
||||
s3_endpoint="http://127.0.0.1:" + os.getenv("TEST_S3_PORT", "59005"),
|
||||
s3_access_key=(ROOT / ("secrets/" + prefix + "_" + access)).read_text().strip(),
|
||||
s3_secret_key=(ROOT / ("secrets/" + prefix + "_" + secret)).read_text().strip(),
|
||||
broker_url="redis://127.0.0.1:" + os.getenv("TEST_BROKER_PORT", "56385") + "/0",
|
||||
job_timeout=int(os.getenv("TEST_JOB_TIMEOUT_SECONDS", "5")),
|
||||
fault_injection=True,
|
||||
)
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# All running application/state services; no migration, down -v, flush or seed.
|
||||
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 broker relay worker lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8084"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose
|
||||
from pipeline_lab import settings
|
||||
|
||||
from messenger.cleanup import collect
|
||||
from messenger.pipeline import Pipeline, intent
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("action", choices=("seed", "verify"))
|
||||
parser.add_argument("--seed", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
|
||||
args = parser.parse_args()
|
||||
config = settings("reconciler")
|
||||
db = PostgresRepository(config.database_url, pool_size=3)
|
||||
db.open()
|
||||
store = ObjectStore(config)
|
||||
try:
|
||||
if args.action == "seed":
|
||||
check(not args.seed.exists(), "Seed already exists")
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
try:
|
||||
c = Client(args.base_url)
|
||||
alice, _, _, room = c.bootstrap()
|
||||
active, data = c.attachment(alice, room)
|
||||
failed, _ = c.attachment(alice, room)
|
||||
orphan = str(uuid4())
|
||||
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
|
||||
uploader = ObjectStore(settings("api"))
|
||||
try:
|
||||
uploader.put(
|
||||
"staging/" + orphan,
|
||||
io.BytesIO(data),
|
||||
len(data),
|
||||
"image/png",
|
||||
hashlib.sha256(data).hexdigest(),
|
||||
)
|
||||
finally:
|
||||
uploader.close()
|
||||
with db.connection(write=True) as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET "
|
||||
"next_attempt_at=clock_timestamp()+interval '1 hour' "
|
||||
"WHERE job_id=%s",
|
||||
(j["job_id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
|
||||
"hour' WHERE job_id=%s",
|
||||
(j["job_id"],),
|
||||
)
|
||||
f = conn.execute(
|
||||
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
|
||||
"(%s,'before_effect','permanent')",
|
||||
(f["job_id"],),
|
||||
)
|
||||
seed = {
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"active_id": active["id"],
|
||||
"failed_id": failed["id"],
|
||||
"orphan_id": orphan,
|
||||
}
|
||||
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
|
||||
"credentials saved"
|
||||
)
|
||||
finally:
|
||||
compose("start", "worker", "relay")
|
||||
else:
|
||||
check(args.output is not None and not args.output.exists(), "Choose a new output file")
|
||||
seed = json.loads(args.seed.read_text())
|
||||
# Real S3 timestamps and failed retention; no modified clocks.
|
||||
while True:
|
||||
with db.connection() as conn:
|
||||
f = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
|
||||
).fetchone()
|
||||
check(f["state"] == "failed", "Failed seed did not reach terminal state")
|
||||
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
|
||||
if age >= 301:
|
||||
break
|
||||
check(age >= 0, "Clock mismatch")
|
||||
time.sleep(min(1, 301 - age))
|
||||
with db.connection() as conn:
|
||||
ready = conn.execute(
|
||||
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
|
||||
).fetchone()
|
||||
check(ready is not None, "Need a published attachment to protect")
|
||||
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
|
||||
before = {key: store.head(key)["ETag"] for key in keys}
|
||||
preview = collect(db, store, 300, False)
|
||||
result = collect(db, store, 300, True)
|
||||
check(
|
||||
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
|
||||
)
|
||||
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
|
||||
check(
|
||||
store.head("staging/" + seed["failed_id"]) is None,
|
||||
"Expired failed source not deleted",
|
||||
)
|
||||
check(
|
||||
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
|
||||
"Published result changed",
|
||||
)
|
||||
with db.connection(write=True) as conn:
|
||||
expired = conn.execute(
|
||||
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
|
||||
(seed["failed_id"],),
|
||||
).fetchone()
|
||||
check(expired["source_expired"], "Expired source not registered")
|
||||
j = conn.execute(
|
||||
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
|
||||
"attachment_id=%s RETURNING job_id",
|
||||
(seed["active_id"],),
|
||||
).fetchone()
|
||||
intent(conn, j["job_id"])
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
if db.attachment(seed["active_id"])["storage_state"] == "ready":
|
||||
break
|
||||
time.sleep(0.2)
|
||||
check(
|
||||
db.attachment(seed["active_id"])["storage_state"] == "ready",
|
||||
"Protected work failed to complete",
|
||||
)
|
||||
proof = {
|
||||
"retention_seconds": 300,
|
||||
"failed_real_age_seconds": age,
|
||||
"seed": seed,
|
||||
"dry_run": preview,
|
||||
"apply": result,
|
||||
"active_protected_then_ready": True,
|
||||
"orphan_deleted": True,
|
||||
"failed_source_expired": True,
|
||||
"published_objects_unchanged": len(keys),
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(proof, indent=2) + "\n")
|
||||
print("PASS: real-age cleanup preserves active work and published results")
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real LB requests during each replica failure and sequential replacement."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
values = sorted(values)
|
||||
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base, actor, room, workers=6):
|
||||
self.base, self.actor, self.room = base, actor, room
|
||||
self.prefix = uuid4().hex
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.executor = ThreadPoolExecutor(max_workers=workers)
|
||||
self.workers = workers
|
||||
|
||||
def worker(self, worker):
|
||||
client, iteration = Client(self.base, timeout=3), 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
write = iteration % 10 == 9
|
||||
try:
|
||||
if write:
|
||||
key = f"{self.prefix}:{worker}:{iteration}"
|
||||
response = client.message(self.actor, self.room, key, key=key)
|
||||
else:
|
||||
response = client.call(
|
||||
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
|
||||
)
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": True,
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
"message": response["json"] if write else None,
|
||||
}
|
||||
iteration += 1
|
||||
except Failure:
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": False,
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
}
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.executor.shutdown(wait=True)
|
||||
for future in self.futures:
|
||||
future.result()
|
||||
self.finished = time.monotonic()
|
||||
|
||||
def report(self, action_start, action_end):
|
||||
events = sorted(self.events, key=lambda row: row["at"])
|
||||
ok = [row for row in events if row["ok"]]
|
||||
after = [row for row in ok if row["at"] >= action_end]
|
||||
check(bool(after), "No successful request after action")
|
||||
elapsed = self.finished - self.started
|
||||
latency = [row["latency_ms"] for row in ok]
|
||||
return {
|
||||
"duration_seconds": elapsed,
|
||||
"workers": self.workers,
|
||||
"requests": len(events),
|
||||
"successes": len(ok),
|
||||
"errors": len(events) - len(ok),
|
||||
"successful_rps": len(ok) / elapsed,
|
||||
"error_fraction": (len(events) - len(ok)) / len(events),
|
||||
"first_success_after_action_seconds": after[0]["at"] - action_start,
|
||||
"action_seconds": action_end - action_start,
|
||||
"p50_ms": percentile(latency, 0.5),
|
||||
"p95_ms": percentile(latency, 0.95),
|
||||
"p99_ms": percentile(latency, 0.99),
|
||||
"instances": sorted({row["instance"] for row in ok}),
|
||||
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
|
||||
"confirmed_writes": sum(row["message"] is not None for row in ok),
|
||||
}
|
||||
|
||||
def verify_messages(self, client, actor, room):
|
||||
actual = history(client, actor, room)
|
||||
by_id = {value["id"]: value for value in actual}
|
||||
check(len(by_id) == len(actual), "History has duplicate IDs")
|
||||
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
|
||||
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
|
||||
for event in self.events:
|
||||
if event["ok"] and event["message"]:
|
||||
check(
|
||||
by_id.get(event["message"]["id"]) == event["message"],
|
||||
"A confirmed message was lost or changed",
|
||||
)
|
||||
|
||||
|
||||
def public_test(target, base):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"make",
|
||||
target,
|
||||
"ACTION_SCRIPT=scripts/"
|
||||
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
|
||||
"BASE_URL=" + base,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=150,
|
||||
)
|
||||
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise SystemExit("Use a new evidence filename")
|
||||
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
|
||||
baseline = instances()
|
||||
check(len(baseline) == 2, "This verification expects the default two API replicas")
|
||||
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
|
||||
wait_lb(args.base_url, 2)
|
||||
result = {"initial_instances": baseline, "failures": []}
|
||||
client = Client(args.base_url, timeout=4)
|
||||
alice, bob, eve, room = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
|
||||
def same_request(_):
|
||||
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
|
||||
return response["json"], response["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
responses = list(pool.map(same_request, range(30)))
|
||||
check(
|
||||
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
|
||||
)
|
||||
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
|
||||
result["concurrent_replays"] = 30
|
||||
result["concurrent_effects"] = 1
|
||||
try:
|
||||
for selected in baseline:
|
||||
selected = instance(selected["id"])
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
|
||||
time.sleep(3)
|
||||
report = traffic.report(action_start, action_end)
|
||||
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
|
||||
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
|
||||
report["stopped"] = selected
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["failures"].append(report)
|
||||
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
print("Verified failure and restoration of one API", flush=True)
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
replaced = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/rolling-update.py"),
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["rolling"] = traffic.report(action_start, action_end)
|
||||
result["rolling"]["replacements"] = json.loads(replaced.stdout)
|
||||
result["rolling"]["confirmed_messages_preserved"] = True
|
||||
print("Verified sequential replacement under traffic", flush=True)
|
||||
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
before = {
|
||||
service: inspect(compose("ps", "-q", service).strip())
|
||||
for service in ("postgres", "valkey", "s3", "broker", "lb")
|
||||
}
|
||||
(ROOT / "evidence/persistence.txt").write_text(
|
||||
public_test("test-persistence", args.base_url)
|
||||
)
|
||||
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
|
||||
check(
|
||||
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
|
||||
"Persistence hook did not restart all state/LB services",
|
||||
)
|
||||
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
|
||||
result["final_instances"] = instances()
|
||||
logs = compose("logs", "--no-color", "api", "lb")
|
||||
check(
|
||||
not any(
|
||||
a["token"] in logs or a["registration"]["password"] in logs
|
||||
for a in (alice, bob, eve)
|
||||
),
|
||||
"Credential appeared in logs",
|
||||
)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: real replica failures, replay, rolling replacement and persistence")
|
||||
finally:
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,513 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from cluster import ROOT, compose, inspect, instances, wait_lb
|
||||
from PIL import Image
|
||||
from pipeline_lab import settings
|
||||
|
||||
from messenger.broker import STREAM, Broker, Job
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import PostgresRepository, stamp
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def until(fn, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = fn()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(0.2)
|
||||
raise AssertionError("Timed out waiting for a durable pipeline condition")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error("Output already exists; choose a new evidence file")
|
||||
config = settings()
|
||||
db = PostgresRepository(config.database_url, pool_size=4)
|
||||
db.open()
|
||||
broker, store = Broker(config), ObjectStore(config)
|
||||
c = Client(args.base_url)
|
||||
alice, bob, eve, room = c.bootstrap()
|
||||
records, accepted = [], []
|
||||
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
def query(sql, params=()):
|
||||
with db.connection() as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def job(a, version=1):
|
||||
return query(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
|
||||
(a["id"], version),
|
||||
)[0]
|
||||
|
||||
def audit(j):
|
||||
return query(
|
||||
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
|
||||
(j["job_id"],),
|
||||
)
|
||||
|
||||
def injected(j, phase):
|
||||
return any(
|
||||
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
|
||||
)
|
||||
|
||||
def fault(j, phase, action, count=1):
|
||||
with db.connection(write=True) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
|
||||
(j["job_id"], phase, action, count),
|
||||
)
|
||||
|
||||
def upload():
|
||||
a, original = c.attachment(alice, room)
|
||||
check(original == data, "Unexpected fixture")
|
||||
accepted.append(a["id"])
|
||||
return a
|
||||
|
||||
def ready(a):
|
||||
value = c.wait_ready(bob, a)
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
|
||||
for v in value["variants"]:
|
||||
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
|
||||
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
|
||||
with Image.open(io.BytesIO(content)) as im:
|
||||
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
|
||||
check(len(value["variants"]) == 3, "Missing variants")
|
||||
check(
|
||||
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
|
||||
"Unexpected duplicate link",
|
||||
)
|
||||
return value
|
||||
|
||||
def saved(name, a, start, **details):
|
||||
j = job(a, details.get("version", 1))
|
||||
current = query(
|
||||
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
|
||||
)[0]
|
||||
keys = [
|
||||
o["Key"]
|
||||
for o in store.client.list_objects_v2(
|
||||
Bucket=store.bucket,
|
||||
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
|
||||
).get("Contents", [])
|
||||
]
|
||||
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
|
||||
record = {
|
||||
"scenario": name,
|
||||
"attachment_id": a["id"],
|
||||
"job_id": str(j["job_id"]),
|
||||
"elapsed_seconds": time.monotonic() - start,
|
||||
"final_state": current["storage_state"],
|
||||
"attempts": j["attempt"],
|
||||
"final_version": current["pipeline_version"],
|
||||
"published_objects": len(keys),
|
||||
"audit": audit(j),
|
||||
**details,
|
||||
}
|
||||
records.append(record)
|
||||
print("PASS: " + name, flush=True)
|
||||
|
||||
def envelope(j, attempt=None):
|
||||
return Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=str(j["attachment_id"]),
|
||||
pipeline_version=j["pipeline_version"],
|
||||
attempt=attempt or max(1, j["attempt"]),
|
||||
created_at=stamp(j["created_at"]),
|
||||
)
|
||||
|
||||
try:
|
||||
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
check(
|
||||
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
||||
"published_at"
|
||||
]
|
||||
is None,
|
||||
"Gap was not established",
|
||||
)
|
||||
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
|
||||
before = instances()
|
||||
compose("restart", "--timeout", "3", "api")
|
||||
wait_lb(args.base_url)
|
||||
after = instances()
|
||||
check(
|
||||
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
|
||||
"API did not restart",
|
||||
)
|
||||
compose("start", "relay")
|
||||
until(
|
||||
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
||||
"published_at"
|
||||
]
|
||||
)
|
||||
entry_ids = [
|
||||
id
|
||||
for id, fields in broker.client.xrange(STREAM)
|
||||
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
|
||||
]
|
||||
check(bool(entry_ids), "Broker has no accepted work")
|
||||
broker_id = compose("ps", "-q", "broker").strip()
|
||||
broker_before = inspect(broker_id)
|
||||
subprocess.run(
|
||||
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
|
||||
)
|
||||
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
|
||||
|
||||
def broker_restored():
|
||||
try:
|
||||
return broker.client.ping()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
until(broker_restored)
|
||||
check(
|
||||
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
|
||||
"Durable stream lost confirmed work",
|
||||
)
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
history = c.call(
|
||||
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
check(message in history["items"], "Queued attachment message lost")
|
||||
saved(
|
||||
"commit_publish_gap_api_restart_broker_sigkill",
|
||||
a,
|
||||
start,
|
||||
api_before=before,
|
||||
api_after=after,
|
||||
broker_before=broker_before,
|
||||
broker_after=inspect(broker_id),
|
||||
broker_entry_ids=entry_ids,
|
||||
message_id=message["id"],
|
||||
)
|
||||
|
||||
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, "after_publish", "crash")
|
||||
compose("start", "relay")
|
||||
until(lambda: injected(j, "after_publish"))
|
||||
until(
|
||||
lambda: (
|
||||
sum(
|
||||
json.loads(f["job"])["job_id"] == str(j["job_id"])
|
||||
for _, f in broker.client.xrange(STREAM)
|
||||
)
|
||||
>= 2
|
||||
)
|
||||
)
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
||||
saved("publish_before_outbox_commit_duplicate", a, start)
|
||||
|
||||
for phase in ("before_effect", "after_effect", "after_commit"):
|
||||
compose("stop", "--timeout", "2", "worker")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, phase, "crash")
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
until(lambda j=j, phase=phase: injected(j, phase))
|
||||
if phase == "after_commit":
|
||||
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
||||
check(job(a)["attempt"] == 1, "Committed job was processed twice")
|
||||
else:
|
||||
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
|
||||
saved("worker_crash_" + phase, a, start)
|
||||
|
||||
for action, count, expected in (
|
||||
("transient", 2, "ready"),
|
||||
("hang", 1, "ready"),
|
||||
("permanent", 1, "failed"),
|
||||
("transient", 5, "failed"),
|
||||
):
|
||||
compose("stop", "--timeout", "2", "worker")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, "before_effect", action, count)
|
||||
compose("start", "worker")
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
if expected == "failed":
|
||||
until(
|
||||
lambda path=path: (
|
||||
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
|
||||
== "failed"
|
||||
)
|
||||
)
|
||||
terminal = job(a)
|
||||
check(terminal["attempt"] == count, "Wrong retry budget")
|
||||
c.call("GET", path + "/content", 409, actor=bob)
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + room["id"] + "/messages",
|
||||
409,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={"Idempotency-Key": uuid4().hex},
|
||||
)
|
||||
# Explicit CLI replay, same logical job; no public administrator endpoint.
|
||||
replay = json.loads(
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.operator",
|
||||
"replay",
|
||||
a["id"],
|
||||
)
|
||||
)
|
||||
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
|
||||
else:
|
||||
terminal = None
|
||||
ready(a)
|
||||
events = audit(j)
|
||||
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
|
||||
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
|
||||
rejected = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.operator",
|
||||
"replay",
|
||||
a["id"],
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
)
|
||||
check(rejected.returncode != 0, "Ready replay was accepted")
|
||||
saved(
|
||||
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
|
||||
a,
|
||||
start,
|
||||
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
|
||||
if terminal
|
||||
else None,
|
||||
backoff_delays=delays,
|
||||
)
|
||||
|
||||
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
old = job(a)
|
||||
old_envelope = envelope(old, 1)
|
||||
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
|
||||
check(outcome == "claimed", "Cannot establish old writer")
|
||||
Pipeline(db, config).replay(a["id"], new_version=True)
|
||||
new = job(a, 2)
|
||||
# Publish v2 first, then v1, plus duplicates through both worker slots.
|
||||
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
|
||||
broker.publish(value)
|
||||
compose("start", "worker", "relay")
|
||||
current = ready(a)
|
||||
row = db.attachment(a["id"])
|
||||
check(
|
||||
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
|
||||
"Old writer overwrote v2",
|
||||
)
|
||||
for _ in range(20):
|
||||
broker.publish(old_envelope)
|
||||
broker.publish(envelope(new, 1))
|
||||
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
|
||||
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
|
||||
check(
|
||||
db.attachment(a["id"])["object_key"] == row["object_key"],
|
||||
"Stale delivery changed current pointer",
|
||||
)
|
||||
check(
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
|
||||
== current,
|
||||
"Old version mutated ready metadata",
|
||||
)
|
||||
saved(
|
||||
"reorder_stale_v1_after_v2_and_publication_fence",
|
||||
a,
|
||||
start,
|
||||
version=2,
|
||||
old_audit=audit(old),
|
||||
duplicates=44,
|
||||
)
|
||||
|
||||
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
|
||||
a = {"id": accepted[0]}
|
||||
for suffix in ("", "/content", "/content?variant=crop"):
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
|
||||
# Direct bucket access and write/delete privilege checks use no public credentials.
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
|
||||
first = db.attachment(accepted[0])
|
||||
unsigned = boto3.client(
|
||||
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
|
||||
)
|
||||
try:
|
||||
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
|
||||
raise AssertionError("Anonymous S3 read succeeded")
|
||||
except ClientError as exc:
|
||||
check(
|
||||
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
|
||||
)
|
||||
finally:
|
||||
unsigned.close()
|
||||
denied = []
|
||||
for role, action in (
|
||||
("api", "put_object"),
|
||||
("api", "delete_object"),
|
||||
("worker", "delete_object"),
|
||||
("reconciler", "put_object"),
|
||||
):
|
||||
scoped = ObjectStore(settings(role))
|
||||
try:
|
||||
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
|
||||
if action == "put_object":
|
||||
kw["Body"] = b"forbidden"
|
||||
try:
|
||||
getattr(scoped.client, action)(**kw)
|
||||
raise AssertionError("IAM action unexpectedly allowed")
|
||||
except ClientError as exc:
|
||||
check(
|
||||
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
|
||||
"Wrong IAM status",
|
||||
)
|
||||
denied.append(role + ":" + action)
|
||||
finally:
|
||||
scoped.close()
|
||||
# Stop the actual uploader, then read original and thumbnail through another replica.
|
||||
response = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
|
||||
202,
|
||||
actor=alice,
|
||||
raw=data,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
cross = response["json"]
|
||||
accepted.append(cross["id"])
|
||||
ready(cross)
|
||||
uploader = response["headers"]["X-Instance-Id"]
|
||||
process = next(v for v in instances() if v["instance"] == uploader)
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
|
||||
)
|
||||
try:
|
||||
wait_lb(args.base_url, count=1)
|
||||
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
|
||||
check(
|
||||
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
|
||||
"Other replica could not serve the original",
|
||||
)
|
||||
c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
|
||||
actor=bob,
|
||||
)
|
||||
cross_proof = {
|
||||
"uploader": uploader,
|
||||
"stopped": inspect(process["id"]),
|
||||
"reader": download["headers"]["X-Instance-Id"],
|
||||
}
|
||||
finally:
|
||||
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url)
|
||||
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
|
||||
check(
|
||||
not any(
|
||||
actor["token"] in logs or actor["registration"]["password"] in logs
|
||||
for actor in (alice, bob, eve)
|
||||
),
|
||||
"Credential in logs",
|
||||
)
|
||||
states = query(
|
||||
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
|
||||
)
|
||||
check(
|
||||
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
|
||||
"Confirmed work lost",
|
||||
)
|
||||
stats = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-c",
|
||||
"import json;from pathlib import Path;print(json.dumps({"
|
||||
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
|
||||
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
|
||||
)
|
||||
result = {
|
||||
"anonymous_s3_status": 403,
|
||||
"denied_iam_actions": denied,
|
||||
"cross_replica": cross_proof,
|
||||
"track": 2,
|
||||
"delivery": "at-least-once",
|
||||
"effect_key": ["attachment_id", "pipeline_version", "variant"],
|
||||
"worker_concurrency": config.worker_concurrency,
|
||||
"prefetch_per_slot": 1,
|
||||
"job_timeout_seconds": config.job_timeout,
|
||||
"lease_seconds": config.lease_seconds,
|
||||
"max_attempts": config.job_max_attempts,
|
||||
"scenarios": records,
|
||||
"accepted": len(accepted),
|
||||
"lost_confirmed": 0,
|
||||
"stale_overwrites": 0,
|
||||
"worker_cgroup": json.loads(stats),
|
||||
"logs_credentials_absent": True,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
|
||||
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
|
||||
finally:
|
||||
compose("start", "worker", "relay", "broker", "api")
|
||||
broker.close()
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""One-process implementation of the lab 1 messenger contract."""
|
||||
@@ -0,0 +1,24 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
def main():
|
||||
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
|
||||
logging.basicConfig(level=level, format="%(message)s")
|
||||
uvicorn.run(
|
||||
"messenger.app:create_app",
|
||||
factory=True,
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
access_log=False,
|
||||
log_level=level.lower(),
|
||||
timeout_keep_alive=2,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,348 @@
|
||||
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from time import perf_counter
|
||||
from typing import Annotated
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from messenger.attachments import Attachments
|
||||
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User, invalid, not_found
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.schemas import (
|
||||
AttachmentView,
|
||||
ChatPage,
|
||||
ChatView,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
MessagePage,
|
||||
MessageView,
|
||||
SessionView,
|
||||
UserView,
|
||||
canonical_uuid,
|
||||
)
|
||||
from messenger.service import Messenger
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class RequestContext:
|
||||
def __init__(self, app: ASGIApp, instance_id: str):
|
||||
self.app = app
|
||||
self.instance_id = instance_id
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
request_id = str(uuid4())
|
||||
scope.setdefault("state", {})["request_id"] = request_id
|
||||
start, status, response_started = perf_counter(), 500, False
|
||||
|
||||
async def with_id(message: Message):
|
||||
nonlocal status, response_started
|
||||
if message["type"] == "http.response.start":
|
||||
response_started = True
|
||||
status = message["status"]
|
||||
message["headers"] = [
|
||||
pair
|
||||
for pair in message.get("headers", [])
|
||||
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
|
||||
] + [
|
||||
(b"x-request-id", request_id.encode()),
|
||||
(b"x-instance-id", self.instance_id.encode()),
|
||||
]
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, with_id)
|
||||
except Exception as exc:
|
||||
# Do not let Uvicorn print exception values that could contain user data.
|
||||
if response_started:
|
||||
raise
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
response = error_response(
|
||||
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
|
||||
)
|
||||
await response(scope, receive, with_id)
|
||||
finally:
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
logger.info(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "request",
|
||||
"request_id": request_id,
|
||||
"method": scope["method"],
|
||||
"route": route,
|
||||
"status": status,
|
||||
"duration_ms": round((perf_counter() - start) * 1000, 3),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request, status: int, code: str, message: str, headers: dict | None = None
|
||||
) -> JSONResponse:
|
||||
identity = getattr(request.state, "request_id", str(uuid4()))
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": identity}},
|
||||
headers={"X-Request-Id": identity, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
settings = settings or Settings.from_env()
|
||||
repository = PostgresRepository(
|
||||
settings.database_url,
|
||||
settings.pool_size,
|
||||
settings.idempotency_ttl,
|
||||
settings.read_autocommit,
|
||||
)
|
||||
cache = SessionCache(settings.cache_url)
|
||||
store = ObjectStore(settings)
|
||||
authentication = AuthService(repository, cache, settings.session_ttl)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
repository.open()
|
||||
app.state.ready = True
|
||||
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.state.ready = False
|
||||
repository.close()
|
||||
cache.close()
|
||||
store.close()
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
|
||||
app.state.messenger = Messenger(repository)
|
||||
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
|
||||
app.state.repository = repository
|
||||
app.state.authentication = authentication
|
||||
app.state.cache = cache
|
||||
app.state.ready = False
|
||||
|
||||
@app.exception_handler(DomainError)
|
||||
async def domain_error(request: Request, exc: DomainError):
|
||||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException):
|
||||
code = "not_found" if exc.status_code == 404 else "invalid_request"
|
||||
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
|
||||
def service(request: Request) -> Messenger:
|
||||
return request.app.state.messenger
|
||||
|
||||
def token_digest(request: Request) -> bytes:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized()
|
||||
return bearer_digest(request.headers.get("authorization"))
|
||||
|
||||
def identity(
|
||||
digest: Annotated[bytes, Depends(token_digest)],
|
||||
) -> Identity:
|
||||
return authentication.identify(digest)
|
||||
|
||||
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
|
||||
return current.user
|
||||
|
||||
async def credentials(request: Request) -> tuple[str, str]:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized("Basic")
|
||||
result = basic_credentials(request.headers.get("authorization"))
|
||||
if await request.body():
|
||||
raise invalid("Вход не принимает тело запроса")
|
||||
return result
|
||||
|
||||
def resource_id(chat_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(chat_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID чата") from None
|
||||
|
||||
def pagination(request: Request) -> tuple[int, str | None]:
|
||||
for key in ("limit", "cursor"):
|
||||
if len(request.query_params.getlist(key)) > 1:
|
||||
raise invalid("Параметр пагинации указан несколько раз")
|
||||
raw = request.query_params.get("limit", "50")
|
||||
try:
|
||||
if not re.fullmatch(r"[0-9]+", raw):
|
||||
raise ValueError
|
||||
limit = int(raw)
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid("limit должен быть целым от 1 до 100") from None
|
||||
return limit, request.query_params.get("cursor")
|
||||
|
||||
def json_input(request: Request):
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
|
||||
|
||||
def message_key(request: Request) -> str:
|
||||
values = request.headers.getlist("idempotency-key")
|
||||
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
|
||||
raise invalid("Требуется корректный Idempotency-Key")
|
||||
return values[0]
|
||||
|
||||
CurrentUser = Annotated[User, Depends(actor)]
|
||||
Service = Annotated[Messenger, Depends(service)]
|
||||
ChatId = Annotated[str, Depends(resource_id)]
|
||||
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.ready:
|
||||
raise DomainError(503, "unavailable", "Приложение не готово")
|
||||
repository.ready()
|
||||
store.ready()
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post(
|
||||
"/api/v1/users",
|
||||
status_code=201,
|
||||
response_model=UserView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def register(body: CreateUser):
|
||||
return authentication.register(body.username, body.display_name, body.password)
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
|
||||
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
|
||||
return authentication.login(*value)
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(current: Annotated[Identity, Depends(identity)]):
|
||||
authentication.logout(current)
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=UserView)
|
||||
def me(current: CurrentUser):
|
||||
return current
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats",
|
||||
status_code=201,
|
||||
response_model=ChatView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
|
||||
return messenger.create_chat(current, body.title, body.member_ids)
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def chats(current: CurrentUser, messenger: Service, page: Pagination):
|
||||
return messenger.chats(current, *page)
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
|
||||
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
|
||||
return messenger.chat(current, chat)
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
status_code=201,
|
||||
response_model=MessageView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def send_message(
|
||||
body: CreateMessage,
|
||||
current: CurrentUser,
|
||||
chat: ChatId,
|
||||
messenger: Service,
|
||||
key: Annotated[str, Depends(message_key)],
|
||||
):
|
||||
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
||||
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
|
||||
return messenger.messages(current, chat, *page)
|
||||
|
||||
def attachment_id(attachment_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(attachment_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID вложения") from None
|
||||
|
||||
AttachmentId = Annotated[str, Depends(attachment_id)]
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentView)
|
||||
async def upload(request: Request, current: CurrentUser, chat: ChatId):
|
||||
return await app.state.attachments.upload(request, current, chat)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
|
||||
def attachment(current: CurrentUser, id: AttachmentId):
|
||||
return app.state.attachments.metadata(current, id)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def content(request: Request, current: CurrentUser, id: AttachmentId):
|
||||
row = app.state.attachments.record(current, id)
|
||||
if row["storage_state"] != "ready":
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
values = request.query_params.getlist("variant")
|
||||
if (
|
||||
len(values) > 1
|
||||
or values
|
||||
and values[0] not in ("original", "thumbnail", "crop", "optimized")
|
||||
):
|
||||
raise invalid("Некорректный вариант вложения")
|
||||
if values and values[0] != "original":
|
||||
row = next((v for v in row["variants"] if v["name"] == values[0]), None)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
row = {**row, "filename": values[0]}
|
||||
body = store.download(row)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := body.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Bounded upload: verified staging and transactional job intent precede HTTP 202."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
from anyio import to_thread
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from messenger.domain import DomainError, invalid, new_id, not_found
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import attachment_value
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def filename(value):
|
||||
if (
|
||||
not value
|
||||
or not value.strip()
|
||||
or len(value) > 255
|
||||
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
|
||||
):
|
||||
raise invalid("Некорректное имя файла")
|
||||
return value
|
||||
|
||||
|
||||
def image_type(file, maximum):
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
if image.width * image.height > maximum:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise ValueError
|
||||
media = "image/png" if image.format == "PNG" else "image/jpeg"
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load() # Reject truncated pixel streams, not just an intact header.
|
||||
file.seek(0)
|
||||
return media
|
||||
except Image.DecompressionBombError, Image.DecompressionBombWarning:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
|
||||
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
|
||||
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, repository, messenger, store, settings):
|
||||
self.repository, self.messenger, self.store = repository, messenger, store
|
||||
self.settings = settings
|
||||
self.slots = asyncio.Semaphore(settings.upload_slots)
|
||||
|
||||
async def upload(self, request, actor, chat_id):
|
||||
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
|
||||
values = request.query_params.getlist("filename")
|
||||
if len(values) != 1:
|
||||
raise invalid("Требуется одно имя файла")
|
||||
name = filename(values[0])
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media not in ("image/png", "image/jpeg"):
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
|
||||
length = request.headers.get("content-length")
|
||||
if length is not None:
|
||||
if not length.isascii() or not length.isdecimal():
|
||||
raise invalid("Некорректный Content-Length")
|
||||
if int(length) > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
try:
|
||||
await asyncio.wait_for(self.slots.acquire(), timeout=5)
|
||||
except TimeoutError:
|
||||
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
digest.update(chunk)
|
||||
await to_thread.run_sync(file.write, chunk)
|
||||
except TimeoutError:
|
||||
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
|
||||
except ClientDisconnect:
|
||||
raise invalid("Загрузка прервана") from None
|
||||
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
|
||||
if actual != media:
|
||||
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
|
||||
id = new_id()
|
||||
key = "staging/" + id
|
||||
await to_thread.run_sync(Pipeline(self.repository, self.settings).reserve, id, key)
|
||||
await to_thread.run_sync(
|
||||
self.store.put, key, file, size, actual, digest.hexdigest()
|
||||
)
|
||||
# A failure before this COMMIT leaves only an unaccepted staging orphan.
|
||||
# The same transaction creates attachment, job and outbox intent.
|
||||
row = await to_thread.run_sync(
|
||||
Pipeline(self.repository, self.settings).accept,
|
||||
id,
|
||||
chat_id,
|
||||
actor.id,
|
||||
name,
|
||||
actual,
|
||||
size,
|
||||
digest.hexdigest(),
|
||||
key,
|
||||
)
|
||||
return attachment_value(row)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def record(self, actor, id):
|
||||
row = self.repository.attachment(id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
self.messenger.chat(actor, str(row["chat_id"]))
|
||||
return row
|
||||
|
||||
def metadata(self, actor, id):
|
||||
return attachment_value(self.record(actor, id))
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from argon2.profiles import RFC_9106_LOW_MEMORY
|
||||
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User
|
||||
from messenger.repository import PostgresRepository
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> DomainError:
|
||||
challenge = scheme + ' realm="messenger"'
|
||||
if scheme == "Basic":
|
||||
challenge += ', charset="UTF-8"'
|
||||
return DomainError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Требуется действительная идентификация",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def basic_credentials(header: str | None) -> tuple[str, str]:
|
||||
try:
|
||||
if header is None or len(header) > 2048:
|
||||
raise ValueError
|
||||
scheme, encoded = header.split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
raise ValueError
|
||||
if not 12 <= len(password) <= 128 or not password.strip():
|
||||
raise ValueError
|
||||
return username, password
|
||||
except ValueError, UnicodeError, binascii.Error:
|
||||
raise unauthorized("Basic") from None
|
||||
|
||||
|
||||
def bearer_digest(header: str | None) -> bytes:
|
||||
if header:
|
||||
parts = header.split(" ")
|
||||
if len(parts) == 2 and parts[0].lower() == "bearer":
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
|
||||
return hashlib.sha256(parts[1].encode()).digest()
|
||||
raise unauthorized()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Identity:
|
||||
user: User
|
||||
token_hash: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: User
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
|
||||
self.repository, self.cache, self.ttl = repository, cache, ttl
|
||||
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
|
||||
self.kdf_slots = BoundedSemaphore(2)
|
||||
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
@contextmanager
|
||||
def kdf(self):
|
||||
if not self.kdf_slots.acquire(timeout=0.5):
|
||||
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.kdf_slots.release()
|
||||
|
||||
def register(self, username: str, display_name: str, password: str) -> User:
|
||||
with self.kdf():
|
||||
encoded = self.hasher.hash(password)
|
||||
return self.repository.add_user(username, display_name, encoded)
|
||||
|
||||
def login(self, username: str, password: str) -> Session:
|
||||
credentials = self.repository.credentials(username)
|
||||
encoded = credentials[1] if credentials else self.dummy_hash
|
||||
try:
|
||||
with self.kdf():
|
||||
self.hasher.verify(encoded, password)
|
||||
except VerificationError, InvalidHashError:
|
||||
raise unauthorized("Basic") from None
|
||||
if credentials is None:
|
||||
raise unauthorized("Basic")
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).digest()
|
||||
user = credentials[0]
|
||||
expires = self.repository.create_session(digest, user.id, self.ttl)
|
||||
return Session(token, "Bearer", expires, user)
|
||||
|
||||
def identify(self, digest: bytes) -> Identity:
|
||||
session = self.repository.active_session(digest)
|
||||
if session is None:
|
||||
raise unauthorized()
|
||||
user_id = str(session["user_id"])
|
||||
user = self.cache.get(digest, user_id)
|
||||
if user is None:
|
||||
user = self.repository.get_user(user_id)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
|
||||
self.cache.put(digest, user, ttl)
|
||||
return Identity(user, digest)
|
||||
|
||||
def logout(self, identity: Identity):
|
||||
self.repository.revoke_session(identity.token_hash)
|
||||
self.cache.delete(identity.token_hash)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
STREAM = "attachment-jobs"
|
||||
GROUP = "image-workers-v1"
|
||||
DLQ = "invalid-jobs"
|
||||
|
||||
|
||||
class Job(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
schema_version: Literal[1]
|
||||
job_id: str
|
||||
type: Literal["attachment.process.v1"]
|
||||
attachment_id: str
|
||||
pipeline_version: int = Field(ge=1)
|
||||
attempt: int = Field(ge=1)
|
||||
created_at: str
|
||||
traceparent: str | None = Field(
|
||||
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
|
||||
)
|
||||
tracestate: str | None = Field(default=None, max_length=512)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def strict_carrier(cls, value):
|
||||
if isinstance(value, dict):
|
||||
if type(value.get("schema_version")) is not int:
|
||||
raise ValueError("Integer schema version required")
|
||||
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
|
||||
raise ValueError("Optional carrier fields must be strings when provided")
|
||||
return value
|
||||
|
||||
@field_validator("job_id", "attachment_id")
|
||||
@classmethod
|
||||
def uuid(cls, value):
|
||||
if str(UUID(value)) != value:
|
||||
raise ValueError("Canonical UUID required")
|
||||
return value
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def timestamp(cls, value):
|
||||
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
|
||||
raise ValueError("RFC3339 required")
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return value
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self, settings):
|
||||
self.client = redis.Redis.from_url(
|
||||
settings.broker_url,
|
||||
decode_responses=True,
|
||||
socket_timeout=2,
|
||||
socket_connect_timeout=1,
|
||||
retry_on_timeout=False,
|
||||
)
|
||||
|
||||
def group(self):
|
||||
try:
|
||||
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
|
||||
def publish(self, job):
|
||||
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
|
||||
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
|
||||
|
||||
def ack(self, id):
|
||||
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
|
||||
self.client.eval(
|
||||
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
|
||||
1,
|
||||
STREAM,
|
||||
GROUP,
|
||||
id,
|
||||
)
|
||||
|
||||
def quarantine(self, id, reason):
|
||||
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
|
||||
with self.client.pipeline(transaction=True) as tx:
|
||||
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
|
||||
tx.xack(STREAM, GROUP, id)
|
||||
tx.xdel(STREAM, id)
|
||||
tx.execute()
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Disposable session-user cache; PostgreSQL always decides token validity."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict
|
||||
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.exceptions import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from messenger.domain import User
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str):
|
||||
self.client = Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
max_connections=8,
|
||||
decode_responses=True,
|
||||
retry_on_timeout=False,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session-user:" + digest.hex()
|
||||
|
||||
def get(self, digest: bytes, user_id: str) -> User | None:
|
||||
try:
|
||||
raw = self.client.get(self.key(digest))
|
||||
if raw:
|
||||
value = User(**json.loads(raw))
|
||||
if value.id == user_id:
|
||||
return value
|
||||
except RedisError, ValueError, TypeError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
|
||||
return None
|
||||
|
||||
def put(self, digest: bytes, user: User, ttl: int):
|
||||
try:
|
||||
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
|
||||
|
||||
def delete(self, digest: bytes):
|
||||
try:
|
||||
self.client.delete(self.key(digest))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Dry-run by default. Retention GC protects all active jobs and every published object."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.pipeline import event
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def collect(db, store, retention, apply=False):
|
||||
if retention < 300:
|
||||
raise ValueError("Retention must be at least 300 seconds")
|
||||
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
|
||||
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
|
||||
with db.connection() as lock:
|
||||
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
|
||||
return {**counts, "busy": True}
|
||||
try:
|
||||
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
|
||||
for obj in page.get("Contents", []):
|
||||
counts["scanned"] += 1
|
||||
key = obj["Key"]
|
||||
parts = key.split("/")
|
||||
try:
|
||||
if (
|
||||
parts[0] not in ("staging", "objects")
|
||||
or str(UUID(parts[1])) != parts[1]
|
||||
):
|
||||
continue
|
||||
except ValueError, IndexError:
|
||||
continue
|
||||
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
|
||||
continue
|
||||
with db.connection(write=True) as conn:
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
|
||||
).fetchone()
|
||||
if (
|
||||
reservation
|
||||
and reservation["state"] == "uploading"
|
||||
and reservation["expires_at"] > datetime.now(UTC)
|
||||
):
|
||||
counts["protected"] += 1
|
||||
continue
|
||||
# This row lock serializes replay/new-version with deletion eligibility.
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
|
||||
).fetchone()
|
||||
active = conn.execute(
|
||||
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
||||
"state IN ('queued','processing') LIMIT 1",
|
||||
(parts[1],),
|
||||
).fetchone()
|
||||
published = (
|
||||
a
|
||||
and a["storage_state"] == "ready"
|
||||
and (
|
||||
key == a["object_key"]
|
||||
or any(v["object_key"] == key for v in a["variants"])
|
||||
)
|
||||
)
|
||||
recent_failed = conn.execute(
|
||||
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
||||
"state='failed' "
|
||||
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
|
||||
(parts[1], retention),
|
||||
).fetchone()
|
||||
if active or published or recent_failed:
|
||||
counts["protected"] += 1
|
||||
continue
|
||||
head = store.head(key)
|
||||
if (
|
||||
not head
|
||||
or head["ETag"] != obj["ETag"]
|
||||
or head["ContentLength"] != obj["Size"]
|
||||
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
|
||||
< retention
|
||||
):
|
||||
continue
|
||||
counts["eligible"] += 1
|
||||
if apply:
|
||||
if reservation and reservation["state"] == "uploading":
|
||||
conn.execute(
|
||||
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
|
||||
(parts[1],),
|
||||
)
|
||||
store.client.delete_object(Bucket=store.bucket, Key=key)
|
||||
counts["deleted"] += 1
|
||||
rows = conn.execute(
|
||||
"UPDATE attachment_jobs SET source_expired=true WHERE "
|
||||
"source_key=%s "
|
||||
"AND state='failed' RETURNING job_id",
|
||||
(key,),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
event(conn, row["job_id"], "source_expired")
|
||||
finally:
|
||||
lock.execute("SELECT pg_advisory_unlock(2301005)")
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Versioned keyset position, bound to a list and its resource, never an offset."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.domain import HistoryKey, invalid
|
||||
|
||||
|
||||
def encode_cursor(scope: str, position: HistoryKey) -> str:
|
||||
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
|
||||
return (
|
||||
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
|
||||
|
||||
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not value or len(value) > 1024:
|
||||
raise ValueError
|
||||
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
|
||||
data = json.loads(raw)
|
||||
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
|
||||
raise ValueError
|
||||
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
|
||||
raise ValueError
|
||||
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
|
||||
raise ValueError
|
||||
stamp = datetime.fromisoformat(data["created_at"])
|
||||
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
|
||||
raise ValueError
|
||||
identity = str(UUID(data["id"]))
|
||||
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
|
||||
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
|
||||
raise invalid("Некорректный cursor или другая область пагинации") from None
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Immutable domain values; no dependency on HTTP or a storage technology."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message: str = "Некорректный запрос") -> DomainError:
|
||||
return DomainError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found() -> DomainError:
|
||||
return DomainError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
status: str = "ready"
|
||||
variants: tuple = ()
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
type HistoryKey = tuple[str, str]
|
||||
|
||||
|
||||
def history_key(item: Chat | Message) -> HistoryKey:
|
||||
return item.created_at, item.id
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Operator-only, single-use fault barriers. Disabled in the normal Compose stack."""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from messenger.domain import DomainError
|
||||
from messenger.pipeline import event
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def checkpoint(db, settings, job, phase):
|
||||
if not settings.fault_injection:
|
||||
return
|
||||
with db.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE pipeline_faults SET remaining=remaining-1 WHERE job_id=%s AND phase=%s "
|
||||
"AND remaining>0 RETURNING action",
|
||||
(job, phase),
|
||||
).fetchone()
|
||||
if row:
|
||||
event(conn, job, "fault_injected", phase=phase, action=row["action"], pid=os.getpid())
|
||||
if not row:
|
||||
return
|
||||
match row["action"]:
|
||||
case "crash":
|
||||
os._exit(87)
|
||||
case "hang":
|
||||
time.sleep(settings.job_timeout * 3)
|
||||
case "transient":
|
||||
raise DomainError(503, "unavailable", "Synthetic transient failure")
|
||||
case "permanent":
|
||||
raise PermanentFailure
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Pure, versioned image transforms. No network, filenames, or EXIF in output."""
|
||||
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
|
||||
def fit(image, maximum):
|
||||
scale = max(image.size)
|
||||
if scale <= maximum:
|
||||
return image.copy()
|
||||
size = tuple(max(1, n * maximum // scale) for n in image.size)
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def variants(source, version):
|
||||
if version not in (1, 2):
|
||||
raise ValueError("Unsupported pipeline version")
|
||||
with Image.open(source) as raw:
|
||||
oriented = ImageOps.exif_transpose(raw)
|
||||
rgba = oriented.convert("RGBA")
|
||||
image = Image.new("RGB", rgba.size, "white")
|
||||
image.paste(rgba, mask=rgba.getchannel("A"))
|
||||
rgba.close()
|
||||
oriented.close()
|
||||
try:
|
||||
for name, output, format, options in (
|
||||
("thumbnail", fit(image, 256), "JPEG", {"quality": 85, "subsampling": 0}),
|
||||
(
|
||||
"crop",
|
||||
ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS),
|
||||
"JPEG",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"optimized",
|
||||
fit(image, 1280),
|
||||
"WEBP",
|
||||
{"quality": 82 if version == 1 else 76, "method": 4},
|
||||
),
|
||||
):
|
||||
with output, io.BytesIO() as buffer:
|
||||
output.save(buffer, format=format, exif=b"", **options)
|
||||
yield (
|
||||
name,
|
||||
buffer.getvalue(),
|
||||
output.size,
|
||||
"image/webp" if format == "WEBP" else "image/jpeg",
|
||||
)
|
||||
finally:
|
||||
image.close()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Transactional, checksummed migrations, serialized across concurrent starts."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from messenger.settings import secret
|
||||
|
||||
|
||||
def migrate(conninfo: str, directory: Path):
|
||||
with psycopg.connect(conninfo, connect_timeout=5) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, "
|
||||
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
|
||||
)
|
||||
for path in sorted(directory.glob("*.sql")):
|
||||
content = path.read_text()
|
||||
checksum = hashlib.sha256(content.encode()).hexdigest()
|
||||
current = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
|
||||
).fetchone()
|
||||
if current:
|
||||
if current[0] != checksum:
|
||||
raise RuntimeError("Applied migration checksum changed: " + path.stem)
|
||||
continue
|
||||
conn.execute(content)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
|
||||
(path.stem, checksum),
|
||||
)
|
||||
print("Applied migration " + path.stem)
|
||||
|
||||
|
||||
def main():
|
||||
migrate(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Explicit local operator actions, never exposed as a public HTTP endpoint."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=("failed", "replay", "new-version"))
|
||||
parser.add_argument("attachment_id", nargs="?")
|
||||
args = parser.parse_args()
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
try:
|
||||
if args.action == "failed":
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT job_id,attachment_id,pipeline_version,attempt,error_code,"
|
||||
"source_expired FROM attachment_jobs WHERE state='failed' "
|
||||
"ORDER BY updated_at LIMIT 100"
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=str))
|
||||
else:
|
||||
if not args.attachment_id:
|
||||
parser.error("attachment_id is required")
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT j.source_key,a.object_key,a.storage_state FROM attachment_jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id AND "
|
||||
"a.pipeline_version=j.pipeline_version "
|
||||
"WHERE a.id=%s",
|
||||
(args.attachment_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
parser.error("Unknown attachment")
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
key = row["object_key"] if row["storage_state"] == "ready" else row["source_key"]
|
||||
if not store.head(key):
|
||||
parser.error("Source is no longer retained")
|
||||
finally:
|
||||
store.close()
|
||||
job = Pipeline(db, settings).replay(
|
||||
args.attachment_id, new_version=args.action == "new-version"
|
||||
)
|
||||
print(json.dumps({"job_id": job, "action": args.action}))
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
|
||||
|
||||
import os
|
||||
import random
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from messenger.domain import DomainError, new_id
|
||||
|
||||
|
||||
def event(conn, job_id, name, **details):
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
|
||||
(job_id, name, Jsonb(details)),
|
||||
)
|
||||
|
||||
|
||||
def intent(conn, job_id, when=None):
|
||||
conn.execute(
|
||||
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
|
||||
(job_id, when),
|
||||
)
|
||||
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, repository, settings):
|
||||
self.db, self.settings = repository, settings
|
||||
|
||||
def reserve(self, id, key):
|
||||
with self.db.connection(write=True) as conn:
|
||||
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
|
||||
|
||||
def accept(self, id, chat, owner, name, media, size, digest, key):
|
||||
with self.db.connection(write=True) as conn:
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
|
||||
).fetchone()
|
||||
if (
|
||||
not reservation
|
||||
or reservation["state"] != "uploading"
|
||||
or reservation["object_key"] != key
|
||||
or reservation["expires_at"] <= datetime.now(UTC)
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
|
||||
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"object_key,storage_state) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
|
||||
(id, chat, owner, name, media, size, digest, key),
|
||||
).fetchone()
|
||||
job = new_id()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
|
||||
"VALUES (%s,%s,1,%s)",
|
||||
(job, id, key),
|
||||
)
|
||||
intent(conn, job)
|
||||
event(conn, job, "accepted")
|
||||
return row
|
||||
|
||||
def claim(self, envelope):
|
||||
with self.db.connection(write=True) as conn:
|
||||
# All writers lock attachment before job, including publication and operator replay.
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
return "unknown", None
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
|
||||
(envelope.job_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
not j
|
||||
or j["attachment_id"] != a["id"]
|
||||
or j["pipeline_version"] != envelope.pipeline_version
|
||||
):
|
||||
return "unknown", None
|
||||
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
|
||||
event(conn, j["job_id"], "stale_delivery")
|
||||
return "stale", None
|
||||
if j["state"] == "ready":
|
||||
event(conn, j["job_id"], "duplicate_ready")
|
||||
return "ready", a
|
||||
if j["state"] == "failed":
|
||||
return "failed", None
|
||||
now = datetime.now(UTC)
|
||||
if j["state"] == "processing" or j["next_attempt_at"] > now:
|
||||
return "busy", None
|
||||
if envelope.attempt != j["attempt"] + 1:
|
||||
# An older envelope cannot steal a scheduled retry or an explicit replay.
|
||||
return "obsolete_attempt", None
|
||||
fence = new_id()
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
|
||||
"lease_until=clock_timestamp()+(%s*interval '1 "
|
||||
"second'),updated_at=clock_timestamp() "
|
||||
"WHERE job_id=%s",
|
||||
(fence, self.settings.lease_seconds, j["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='processing',"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(a["id"],),
|
||||
)
|
||||
event(
|
||||
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
|
||||
)
|
||||
return "claimed", {
|
||||
**a,
|
||||
"job_id": str(j["job_id"]),
|
||||
"source_key": j["source_key"],
|
||||
"fence": fence,
|
||||
"attempt": j["attempt"] + 1,
|
||||
}
|
||||
|
||||
def publish(self, row, original_key, variants):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
||||
).fetchone()
|
||||
if not self.owns(a, j, row):
|
||||
event(conn, row["job_id"], "publication_fenced")
|
||||
return False
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
|
||||
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(original_key, Jsonb(variants), row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE job_id=%s",
|
||||
(row["job_id"],),
|
||||
)
|
||||
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def owns(a, j, row):
|
||||
return (
|
||||
a
|
||||
and j
|
||||
and a["pipeline_version"] == row["pipeline_version"]
|
||||
and j["state"] == "processing"
|
||||
and str(j["fence"]) == row["fence"]
|
||||
and j["lease_until"] > datetime.now(UTC)
|
||||
)
|
||||
|
||||
def failure(self, row, code, permanent=False):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
||||
).fetchone()
|
||||
if self.owns(a, j, row):
|
||||
self._retry(conn, j, code, permanent)
|
||||
|
||||
def _retry(self, conn, j, code, permanent=False):
|
||||
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
|
||||
state = "failed" if failed else "queued"
|
||||
if failed and not permanent:
|
||||
code = "retry_exhausted"
|
||||
# Full jitter; durable absolute time survives worker/relay restarts.
|
||||
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
|
||||
due = datetime.now(UTC) + timedelta(seconds=delay)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE job_id=%s",
|
||||
(state, code, due, j["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s",
|
||||
(state, code if failed else None, j["attachment_id"]),
|
||||
)
|
||||
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
|
||||
if not failed:
|
||||
intent(conn, j["job_id"], due)
|
||||
|
||||
def recover(self):
|
||||
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
|
||||
with self.db.connection(write=True) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
|
||||
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
|
||||
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
|
||||
"FOR UPDATE OF a SKIP LOCKED"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
j = conn.execute(
|
||||
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
|
||||
"UPDATE OF j",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
|
||||
event(conn, j["job_id"], "lease_expired")
|
||||
self._retry(conn, j, "worker_timeout")
|
||||
|
||||
def replay(self, attachment, *, new_version=False):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
raise ValueError("Unknown attachment")
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
|
||||
"AND pipeline_version=%s FOR UPDATE",
|
||||
(attachment, a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if new_version:
|
||||
if a["pipeline_version"] != 1:
|
||||
raise ValueError("Only pipeline v1 -> v2 is implemented")
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
|
||||
"AND state IN ('queued','processing','failed')",
|
||||
(j["job_id"],),
|
||||
)
|
||||
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
|
||||
job = new_id()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
|
||||
"VALUES (%s,%s,2,%s)",
|
||||
(job, attachment, key),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET "
|
||||
"pipeline_version=2,storage_state='queued',variants='[]',"
|
||||
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment,),
|
||||
)
|
||||
event(conn, job, "new_version", previous_job=str(j["job_id"]))
|
||||
else:
|
||||
if j["state"] != "failed" or j["source_expired"]:
|
||||
raise ValueError("Replay requires failed job with retained source")
|
||||
job = str(j["job_id"])
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET "
|
||||
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
|
||||
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
|
||||
"job_id=%s",
|
||||
(job,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment,),
|
||||
)
|
||||
event(conn, job, "operator_replay")
|
||||
intent(conn, job)
|
||||
return job
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Transactional outbox publisher and expired-lease watchdog, independently restartable."""
|
||||
|
||||
import signal
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
|
||||
from messenger.broker import Broker, Job
|
||||
from messenger.domain import DomainError
|
||||
from messenger.faults import checkpoint
|
||||
from messenger.pipeline import Pipeline, event
|
||||
from messenger.repository import PostgresRepository, stamp
|
||||
from messenger.settings import Settings
|
||||
|
||||
|
||||
def publish_one(db, broker, settings):
|
||||
with db.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT o.job_id,j.attachment_id,j.pipeline_version,j.attempt,j.created_at "
|
||||
"FROM job_outbox o JOIN attachment_jobs j USING(job_id) "
|
||||
"WHERE o.published_at IS NULL AND o.due_at<=clock_timestamp() AND j.state='queued' "
|
||||
"ORDER BY o.due_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
job = Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(row["job_id"]),
|
||||
attachment_id=str(row["attachment_id"]),
|
||||
pipeline_version=row["pipeline_version"],
|
||||
attempt=row["attempt"] + 1,
|
||||
created_at=stamp(row["created_at"]),
|
||||
)
|
||||
stream_id = broker.publish(job)
|
||||
# Commit-on-return may be lost here; repeat sends the same logical job_id.
|
||||
checkpoint(db, settings, row["job_id"], "after_publish")
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=clock_timestamp(),stream_id=%s WHERE job_id=%s",
|
||||
(stream_id, row["job_id"]),
|
||||
)
|
||||
event(conn, row["job_id"], "published", stream_id=stream_id, attempt=job.attempt)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=4)
|
||||
db.open()
|
||||
broker = Broker(settings)
|
||||
pipeline = Pipeline(db, settings)
|
||||
running = True
|
||||
|
||||
def stop(*_):
|
||||
nonlocal running
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while running:
|
||||
try:
|
||||
pipeline.recover()
|
||||
sent = publish_one(db, broker, settings)
|
||||
Path("/tmp/relay-heartbeat").touch()
|
||||
if not sent:
|
||||
time.sleep(0.2)
|
||||
except redis.RedisError, DomainError:
|
||||
# Keep the transaction uncommitted and retry. No credentials/SQL in logs.
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
broker.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,339 @@
|
||||
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from messenger.domain import (
|
||||
Attachment,
|
||||
Chat,
|
||||
DomainError,
|
||||
HistoryKey,
|
||||
Message,
|
||||
User,
|
||||
invalid,
|
||||
new_id,
|
||||
)
|
||||
|
||||
|
||||
def stamp(value: datetime) -> str:
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def user_value(row) -> User:
|
||||
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
|
||||
|
||||
|
||||
def chat_value(row) -> Chat:
|
||||
return Chat(
|
||||
str(row["id"]),
|
||||
row["title"],
|
||||
tuple(str(x) for x in row["member_ids"]),
|
||||
stamp(row["created_at"]),
|
||||
)
|
||||
|
||||
|
||||
def message_value(row) -> Message:
|
||||
return Message(
|
||||
str(row["id"]),
|
||||
str(row["chat_id"]),
|
||||
str(row["sender_id"]),
|
||||
row["text"],
|
||||
stamp(row["created_at"]),
|
||||
tuple(str(x) for x in row.get("attachment_ids", ())),
|
||||
)
|
||||
|
||||
|
||||
def attachment_value(row) -> Attachment:
|
||||
return Attachment(
|
||||
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
|
||||
row["filename"],
|
||||
row["content_type"],
|
||||
row["size_bytes"],
|
||||
row["sha256"],
|
||||
stamp(row["created_at"]),
|
||||
stamp(row["updated_at"]),
|
||||
row["storage_state"],
|
||||
tuple(
|
||||
{k: v for k, v in item.items() if k != "object_key"} for item in row.get("variants", [])
|
||||
),
|
||||
row.get("error_code"),
|
||||
)
|
||||
|
||||
|
||||
class Repository(Protocol):
|
||||
def get_user(self, user_id: str) -> User | None: ...
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
|
||||
def get_chat(self, chat_id: str) -> Chat | None: ...
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message: ...
|
||||
def page_messages(
|
||||
self, chat_id: str, after: HistoryKey | None, limit: int
|
||||
) -> list[Message]: ...
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
|
||||
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
|
||||
|
||||
|
||||
class PostgresRepository:
|
||||
def __init__(
|
||||
self,
|
||||
conninfo: str,
|
||||
pool_size: int = 8,
|
||||
idempotency_ttl: int = 86400,
|
||||
read_autocommit: bool = True,
|
||||
):
|
||||
self.read_autocommit = read_autocommit
|
||||
self.idempotency_ttl = idempotency_ttl
|
||||
self.pool = ConnectionPool(
|
||||
conninfo,
|
||||
min_size=1,
|
||||
max_size=pool_size,
|
||||
timeout=1,
|
||||
max_waiting=32,
|
||||
open=False,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"autocommit": True,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
|
||||
},
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=15)
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=5)
|
||||
|
||||
@contextmanager
|
||||
def connection(self, *, write: bool = False):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
transaction = (
|
||||
conn.transaction() if write or not self.read_autocommit else nullcontext()
|
||||
)
|
||||
with transaction:
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
errors.QueryCanceled,
|
||||
errors.LockNotAvailable,
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
|
||||
|
||||
def ready(self):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS n FROM schema_migrations "
|
||||
"WHERE version IN "
|
||||
"('001_initial','002_idempotency','003_attachments',"
|
||||
"'004_pipeline','005_upload_handoff')"
|
||||
).fetchone()
|
||||
if row["n"] != 5:
|
||||
raise DomainError(503, "unavailable", "Схема ещё не готова")
|
||||
|
||||
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (id,username,display_name,password_hash) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return user_value(row)
|
||||
except errors.UniqueViolation:
|
||||
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
|
||||
|
||||
def credentials(self, username: str):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
return (user_value(row), row["password_hash"]) if row else None
|
||||
|
||||
def get_user(self, user_id: str) -> User | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||
return user_value(row) if row else None
|
||||
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||
[(row["id"], member, i) for i, member in enumerate(member_ids)],
|
||||
)
|
||||
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
|
||||
except errors.ForeignKeyViolation:
|
||||
raise invalid("Указан неизвестный участник") from None
|
||||
|
||||
def get_chat(self, chat_id: str) -> Chat | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
|
||||
).fetchone()
|
||||
return chat_value(row) if row else None
|
||||
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
|
||||
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (user_id, *after, limit) if after else (user_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
|
||||
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
|
||||
+ condition
|
||||
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [chat_value(row) for row in rows]
|
||||
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
return message_value(row)
|
||||
|
||||
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
|
||||
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (chat_id, *after, limit) if after else (chat_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.chat_id=%s"
|
||||
+ condition
|
||||
+ " ORDER BY m.created_at,m.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [message_value(row) for row in rows]
|
||||
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message:
|
||||
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
payload = json.dumps(
|
||||
{"text": text, "attachment_ids": list(attachment_ids)},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
digest = hashlib.sha256(payload.encode()).digest()
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
previous = conn.execute(
|
||||
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
|
||||
"JOIN messages m ON m.id=r.response_message_id "
|
||||
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
|
||||
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
|
||||
(sender_id, chat_id, key),
|
||||
).fetchone()
|
||||
if previous:
|
||||
if previous["payload_hash"] != digest:
|
||||
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
|
||||
return message_value(previous)
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
|
||||
"AND chat_id=%s AND idempotency_key=%s",
|
||||
(sender_id, chat_id, key),
|
||||
)
|
||||
if attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
|
||||
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
|
||||
(chat_id, list(attachment_ids)),
|
||||
).fetchall()
|
||||
if len(rows) != len(attachment_ids):
|
||||
raise DomainError(404, "not_found", "Объект не найден")
|
||||
if any(row["storage_state"] == "failed" for row in rows):
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
message = message_value(
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
)
|
||||
for position, attachment in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
|
||||
(message.id, chat_id, attachment, position),
|
||||
)
|
||||
message = Message(
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at,
|
||||
attachment_ids,
|
||||
)
|
||||
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
|
||||
return message
|
||||
|
||||
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests "
|
||||
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
|
||||
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
|
||||
)
|
||||
|
||||
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO sessions (token_hash,user_id,expires_at) "
|
||||
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
|
||||
(token_hash, user_id, ttl),
|
||||
).fetchone()
|
||||
return stamp(row["expires_at"])
|
||||
|
||||
def active_session(self, token_hash: bytes):
|
||||
with self.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
|
||||
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
|
||||
def revoke_session(self, token_hash: bytes):
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
|
||||
|
||||
def attachment(self, id):
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Strict lab 5 wire types; password never belongs to a response User."""
|
||||
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def canonical_uuid(value: str) -> str:
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
|
||||
raise ValueError("Ожидается UUID")
|
||||
return str(UUID(value))
|
||||
|
||||
|
||||
def nonblank(value: str) -> str:
|
||||
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
|
||||
raise ValueError("Ожидается непустая строка Unicode")
|
||||
return value
|
||||
|
||||
|
||||
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
|
||||
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
|
||||
|
||||
|
||||
class Input(BaseModel):
|
||||
model_config = ConfigDict(strict=True, extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(Input):
|
||||
username: str = Field(min_length=3, max_length=32)
|
||||
display_name: ShortText
|
||||
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def valid_username(cls, value: str) -> str:
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Недопустимое имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(Input):
|
||||
title: ShortText
|
||||
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class CreateMessage(Input):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_content(self):
|
||||
if (
|
||||
not self.text.strip()
|
||||
and not self.attachment_ids
|
||||
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
|
||||
or len(set(self.attachment_ids)) != len(self.attachment_ids)
|
||||
):
|
||||
raise ValueError("Некорректное содержимое сообщения")
|
||||
return self
|
||||
|
||||
|
||||
class View(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, extra="forbid")
|
||||
|
||||
|
||||
class UserView(View):
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SessionView(View):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: UserView
|
||||
|
||||
|
||||
class ChatView(View):
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class MessageView(View):
|
||||
attachment_ids: list[str]
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(View):
|
||||
items: list[ChatView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class MessagePage(View):
|
||||
items: list[MessageView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class AttachmentView(View):
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from messenger.cursor import decode_cursor, encode_cursor
|
||||
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
|
||||
from messenger.repository import Repository
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Page[T]:
|
||||
items: list[T]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class Messenger:
|
||||
def __init__(self, repository: Repository):
|
||||
self.repository = repository
|
||||
|
||||
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
|
||||
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
|
||||
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
|
||||
return self.repository.add_chat(title, tuple(member_ids))
|
||||
|
||||
def chat(self, actor: User, chat_id: str) -> Chat:
|
||||
chat = self.repository.get_chat(chat_id)
|
||||
if chat is None or actor.id not in chat.member_ids:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
|
||||
scope = "chats:" + actor.id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
|
||||
|
||||
def send(
|
||||
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
|
||||
) -> Message:
|
||||
self.chat(actor, chat_id)
|
||||
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
|
||||
|
||||
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
|
||||
self.chat(actor, chat_id)
|
||||
scope = "messages:" + chat_id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
|
||||
|
||||
@staticmethod
|
||||
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
|
||||
visible = items[:limit]
|
||||
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
|
||||
return Page(visible, cursor)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Explicit configuration; credentials are read from untracked secret files."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str, default: str, optional=False) -> str:
|
||||
path = Path(os.getenv(name, default))
|
||||
return path.read_text().strip() if path.exists() or not optional else ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = field(repr=False)
|
||||
cache_url: str = field(repr=False)
|
||||
session_ttl: int = 3600
|
||||
pool_size: int = 8
|
||||
idempotency_ttl: int = 86400
|
||||
read_autocommit: bool = True
|
||||
|
||||
s3_endpoint: str = "http://127.0.0.1:59005"
|
||||
s3_region: str = "us-east-1"
|
||||
s3_bucket: str = "messenger-attachments"
|
||||
s3_access_key: str = field(default="", repr=False)
|
||||
s3_secret_key: str = field(default="", repr=False)
|
||||
max_upload_bytes: int = 10485760
|
||||
max_image_pixels: int = 40000000
|
||||
upload_slots: int = 1
|
||||
|
||||
broker_url: str = field(default="redis://broker:6379/0", repr=False)
|
||||
worker_concurrency: int = 2
|
||||
job_max_attempts: int = 5
|
||||
job_timeout: int = 20
|
||||
staging_retention: int = 86400
|
||||
fault_injection: bool = False
|
||||
|
||||
@property
|
||||
def lease_seconds(self):
|
||||
return self.job_timeout + 3
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
|
||||
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
if idempotency_ttl < 86400:
|
||||
raise ValueError("Idempotency retention must be at least 24 hours")
|
||||
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
|
||||
raise ValueError("Invalid session lifetime or connection pool size")
|
||||
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
|
||||
raise ValueError("Invalid upload limits")
|
||||
concurrency = int(os.getenv("WORKER_CONCURRENCY", "2"))
|
||||
attempts = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
timeout = int(os.getenv("JOB_TIMEOUT_SECONDS", "20"))
|
||||
retention = int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))
|
||||
if not 1 <= concurrency <= 4 or not 1 <= attempts <= 10 or not 2 <= timeout <= 120:
|
||||
raise ValueError("Invalid worker limits")
|
||||
if retention < 300:
|
||||
raise ValueError("Staging retention must be at least 300 seconds")
|
||||
return cls(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
|
||||
ttl,
|
||||
size,
|
||||
idempotency_ttl,
|
||||
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
|
||||
os.getenv("S3_ENDPOINT", "http://s3:9000"),
|
||||
os.getenv("S3_REGION", "us-east-1"),
|
||||
os.getenv("S3_BUCKET", "messenger-attachments"),
|
||||
secret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id", optional=True),
|
||||
secret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key", optional=True),
|
||||
maximum,
|
||||
pixels,
|
||||
broker_url=secret("BROKER_URL_FILE", "/run/secrets/broker_url")
|
||||
if os.path.exists(os.getenv("BROKER_URL_FILE", "/run/secrets/broker_url"))
|
||||
else "redis://broker:6379/0",
|
||||
worker_concurrency=concurrency,
|
||||
job_max_attempts=attempts,
|
||||
job_timeout=timeout,
|
||||
staging_retention=retention,
|
||||
fault_injection=os.getenv("ENABLE_FAULT_INJECTION", "0") == "1",
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from messenger.domain import DomainError
|
||||
|
||||
|
||||
def unavailable():
|
||||
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, settings):
|
||||
self.bucket = settings.s3_bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key,
|
||||
aws_secret_access_key=settings.s3_secret_key,
|
||||
config=Config(
|
||||
connect_timeout=0.5,
|
||||
read_timeout=2,
|
||||
max_pool_connections=8,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def put(self, key, file, size, media, digest):
|
||||
try:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=file,
|
||||
ContentLength=size,
|
||||
ContentType=media,
|
||||
Metadata={"sha256": digest},
|
||||
)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def head(self, key):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if str(exc.response.get("Error", {}).get("Code")) in ("404", "NoSuchKey", "NotFound"):
|
||||
return None
|
||||
raise unavailable() from None
|
||||
except BotoCoreError:
|
||||
raise unavailable() from None
|
||||
|
||||
def download(self, row):
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
|
||||
if response["ContentLength"] != row["size_bytes"]:
|
||||
response["Body"].close()
|
||||
raise unavailable()
|
||||
return response["Body"]
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,200 @@
|
||||
"""Bounded supervisor: one disposable OS process per job, hard wall-clock and RAM limits."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import multiprocessing
|
||||
import resource
|
||||
import signal
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from pydantic import ValidationError
|
||||
|
||||
from messenger.attachments import image_type
|
||||
from messenger.broker import GROUP, STREAM, Broker, Job
|
||||
from messenger.domain import DomainError
|
||||
from messenger.faults import PermanentFailure, checkpoint
|
||||
from messenger.images import variants
|
||||
from messenger.pipeline import Pipeline, event
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def verify_effect(store, row):
|
||||
records = [row, *row["variants"]]
|
||||
if len(records) != 4 or {v["name"] for v in row["variants"]} != {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}:
|
||||
raise DomainError(503, "unavailable", "Incomplete result")
|
||||
for item in records:
|
||||
head = store.head(item["object_key"])
|
||||
if (
|
||||
not head
|
||||
or head["ContentLength"] != item["size_bytes"]
|
||||
or head["Metadata"].get("sha256") != item["sha256"]
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Incomplete result")
|
||||
|
||||
|
||||
def handle(db, broker, store, settings, id, payload):
|
||||
try:
|
||||
job = Job.model_validate_json(payload)
|
||||
except ValidationError:
|
||||
broker.quarantine(id, "invalid_envelope")
|
||||
return
|
||||
pipeline = Pipeline(db, settings)
|
||||
outcome, row = pipeline.claim(job)
|
||||
if outcome == "busy":
|
||||
return # Leave pending. Watchdog recovers the DB lease and schedules a retry.
|
||||
if outcome == "unknown":
|
||||
broker.quarantine(id, "unknown_job")
|
||||
return
|
||||
if outcome == "ready":
|
||||
verify_effect(store, row)
|
||||
broker.ack(id)
|
||||
return
|
||||
if outcome != "claimed":
|
||||
broker.ack(id) # Failed registry / obsolete version / durable retry owns this work.
|
||||
return
|
||||
try:
|
||||
checkpoint(db, settings, job.job_id, "before_effect")
|
||||
with tempfile.TemporaryFile() as original:
|
||||
body = store.download({**row, "object_key": row["source_key"]})
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
while chunk := body.read(65536):
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise PermanentFailure
|
||||
digest.update(chunk)
|
||||
original.write(chunk)
|
||||
finally:
|
||||
body.close()
|
||||
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
||||
raise PermanentFailure
|
||||
image_type(original, settings.max_image_pixels)
|
||||
prefix = f"objects/{row['id']}/v{row['pipeline_version']}/"
|
||||
original_key = prefix + "original"
|
||||
store.put(original_key, original, size, row["content_type"], digest.hexdigest())
|
||||
checkpoint(db, settings, job.job_id, "after_effect")
|
||||
records = []
|
||||
for name, data, (width, height), media in variants(original, row["pipeline_version"]):
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
key = prefix + name
|
||||
store.put(key, io.BytesIO(data), len(data), media, checksum)
|
||||
records.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": media,
|
||||
"size_bytes": len(data),
|
||||
"sha256": checksum,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"object_key": key,
|
||||
}
|
||||
)
|
||||
with db.connection(write=True) as conn:
|
||||
event(conn, job.job_id, "objects_written", version=row["pipeline_version"])
|
||||
committed = pipeline.publish(row, original_key, records)
|
||||
if committed:
|
||||
checkpoint(db, settings, job.job_id, "after_commit")
|
||||
broker.ack(id)
|
||||
except PermanentFailure:
|
||||
pipeline.failure(row, "invalid_image", permanent=True)
|
||||
broker.ack(id)
|
||||
except MemoryError:
|
||||
pipeline.failure(row, "resource_limit", permanent=True)
|
||||
broker.ack(id)
|
||||
except DomainError as exc:
|
||||
pipeline.failure(
|
||||
row,
|
||||
"invalid_image" if exc.status in (413, 415) else "storage_unavailable",
|
||||
permanent=exc.status in (413, 415),
|
||||
)
|
||||
broker.ack(id)
|
||||
except redis.RedisError:
|
||||
# Result may already be committed; retain the pending entry for a verified no-op.
|
||||
raise
|
||||
except Exception:
|
||||
pipeline.failure(row, "processing_failed")
|
||||
broker.ack(id)
|
||||
|
||||
|
||||
def child(settings, id, payload):
|
||||
# Linux hard limit includes Pillow buffers and native allocations. Supervisor kills hangs.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
broker, store = Broker(settings), ObjectStore(settings)
|
||||
try:
|
||||
handle(db, broker, store, settings, id, payload)
|
||||
except redis.RedisError, DomainError:
|
||||
pass # Do not emit secret-bearing driver tracebacks; leave pending / lease intact.
|
||||
finally:
|
||||
store.close()
|
||||
broker.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
settings = Settings.from_env()
|
||||
broker = Broker(settings)
|
||||
consumer, cursor = uuid4().hex, "0-0"
|
||||
context = multiprocessing.get_context("spawn")
|
||||
active, running = [], True
|
||||
|
||||
def stop(*_):
|
||||
nonlocal running
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while running:
|
||||
for process, deadline in active[:]:
|
||||
if not process.is_alive() or time.monotonic() > deadline:
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
active.remove((process, deadline))
|
||||
process.close()
|
||||
Path("/tmp/worker-heartbeat").touch()
|
||||
if len(active) >= settings.worker_concurrency:
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
try:
|
||||
broker.group()
|
||||
reclaimed = broker.client.xautoclaim(
|
||||
STREAM, GROUP, consumer, settings.lease_seconds * 1000, cursor, count=1
|
||||
)
|
||||
cursor, entries = reclaimed[0], reclaimed[1]
|
||||
if not entries:
|
||||
result = broker.client.xreadgroup(
|
||||
GROUP, consumer, {STREAM: ">"}, count=1, block=200
|
||||
)
|
||||
entries = result[0][1] if result else []
|
||||
if entries:
|
||||
id, fields = entries[0]
|
||||
process = context.Process(
|
||||
target=child, args=(settings, id, fields.get("job", ""))
|
||||
)
|
||||
process.start()
|
||||
active.append((process, time.monotonic() + settings.job_timeout))
|
||||
except redis.RedisError:
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
for process, _ in active:
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
broker.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from psycopg import sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.app import create_app
|
||||
from messenger.migrate import migrate
|
||||
from messenger.settings import Settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PASSWORD = "Учебный:пароль😀-12345"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def settings():
|
||||
password = (ROOT / "secrets/postgres_password").read_text().strip()
|
||||
admin = make_conninfo(
|
||||
host="127.0.0.1", port=55437, user="messenger", password=password, dbname="messenger"
|
||||
)
|
||||
database = "student_tests_" + uuid4().hex
|
||||
with psycopg.connect(admin, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
|
||||
config = Settings(
|
||||
make_conninfo(admin, dbname=database),
|
||||
"redis://127.0.0.1:56384/1",
|
||||
broker_url="redis://127.0.0.1:56385/1",
|
||||
s3_bucket="messenger-attachments-tests",
|
||||
s3_access_key=(ROOT / "secrets/s3_test_access_key_id").read_text().strip(),
|
||||
s3_secret_key=(ROOT / "secrets/s3_test_secret_access_key").read_text().strip(),
|
||||
)
|
||||
migrate(config.database_url, ROOT / "migrations")
|
||||
with pytest.MonkeyPatch.context() as environment:
|
||||
environment.setenv("S3_ENDPOINT", config.s3_endpoint)
|
||||
environment.setenv("S3_BUCKET", config.s3_bucket)
|
||||
environment.setenv("S3_ACCESS_KEY_ID_FILE", str(ROOT / "secrets/s3_test_access_key_id"))
|
||||
environment.setenv(
|
||||
"S3_SECRET_ACCESS_KEY_FILE", str(ROOT / "secrets/s3_test_secret_access_key")
|
||||
)
|
||||
yield config
|
||||
with psycopg.connect(admin, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(settings):
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
|
||||
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
|
||||
client.app.state.cache.client.flushdb()
|
||||
yield client
|
||||
|
||||
|
||||
def basic(username, password=PASSWORD):
|
||||
return {
|
||||
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def actors(client):
|
||||
result = []
|
||||
for name in ("alice", "bob", "eve"):
|
||||
r = client.post(
|
||||
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
|
||||
)
|
||||
assert r.status_code == 201
|
||||
session = client.post("/api/v1/auth/sessions", headers=basic(name))
|
||||
assert session.status_code == 201
|
||||
result.append({**r.json(), "token": session.json()["token"]})
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def room(client, actors):
|
||||
response = client.post(
|
||||
"/api/v1/chats",
|
||||
headers={"Authorization": "Bearer " + actors[0]["token"]},
|
||||
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
return response.json()
|
||||
@@ -0,0 +1,299 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from conftest import PASSWORD, ROOT, basic
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from messenger.app import create_app
|
||||
from messenger.auth import basic_credentials
|
||||
from messenger.migrate import migrate
|
||||
|
||||
|
||||
def headers(actor):
|
||||
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
|
||||
|
||||
|
||||
def error(response, status, code):
|
||||
assert response.status_code == status, response.text
|
||||
body = response.json()["error"]
|
||||
assert set(body) == {"code", "message", "request_id"}
|
||||
assert body["code"] == code
|
||||
assert body["request_id"] == response.headers["X-Request-Id"]
|
||||
assert len(response.headers.get_list("X-Request-Id")) == 1
|
||||
if status == 401:
|
||||
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
|
||||
def test_malformed_json(client, body):
|
||||
error(
|
||||
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"username": "ab"},
|
||||
{"username": "a" * 33},
|
||||
{"username": "Upper"},
|
||||
{"username": "кириллица"},
|
||||
{"username": "valid\n"},
|
||||
{"username": 123},
|
||||
{"username": True},
|
||||
{"display_name": 123},
|
||||
{"display_name": None},
|
||||
{"display_name": " \n\t"},
|
||||
{"display_name": "😀" * 101},
|
||||
{"display_name": "\ud800"},
|
||||
{"password": "short"},
|
||||
{"password": "😀" * 129},
|
||||
{"password": 123},
|
||||
{"password": " " * 12},
|
||||
{"password": "\ud800" * 12},
|
||||
{"extra": True},
|
||||
],
|
||||
)
|
||||
def test_strict_registration(client, change):
|
||||
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
|
||||
response = client.post(
|
||||
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
|
||||
)
|
||||
error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_unicode_password_hash_and_authentication(client, settings, actors):
|
||||
for u in actors:
|
||||
assert "password" not in u and "password_hash" not in u
|
||||
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
|
||||
0
|
||||
]
|
||||
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
|
||||
assert (
|
||||
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
|
||||
== 32
|
||||
)
|
||||
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
|
||||
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
|
||||
error(bad, 401, "unauthorized")
|
||||
error(absent, 401, "unauthorized")
|
||||
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
|
||||
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
|
||||
for value in (
|
||||
{},
|
||||
basic("alice"),
|
||||
{"X-User-Id": actors[0]["id"]},
|
||||
{"Authorization": "Bearer unknown"},
|
||||
):
|
||||
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
|
||||
actual = client.get(
|
||||
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
|
||||
).json()
|
||||
assert actual["id"] == actors[0]["id"]
|
||||
error(
|
||||
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
|
||||
def test_invalid_basic(value):
|
||||
with pytest.raises(Exception) as caught:
|
||||
basic_credentials(value)
|
||||
assert caught.value.status == 401
|
||||
|
||||
|
||||
def test_registration_race(client, settings):
|
||||
def register(_):
|
||||
return client.post(
|
||||
"/api/v1/users",
|
||||
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
|
||||
).status_code
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
|
||||
path = "/api/v1/chats/" + room["id"]
|
||||
for suffix in ("", "/messages"):
|
||||
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
||||
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
|
||||
error(
|
||||
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
|
||||
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
|
||||
error(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "rollback", "member_ids": members},
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
|
||||
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
|
||||
for body in (
|
||||
{"text": ""},
|
||||
{"text": " \t"},
|
||||
{"text": "😀" * 4001},
|
||||
{"text": 123},
|
||||
{"text": None},
|
||||
{"text": "ok", "sender_id": actors[1]["id"]},
|
||||
):
|
||||
error(
|
||||
client.post(path + "/messages", headers=headers(actors[0]), json=body),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
for query in (
|
||||
"limit=0",
|
||||
"limit=101",
|
||||
"limit=true",
|
||||
"limit=1.0",
|
||||
"limit=1&limit=2",
|
||||
"cursor=",
|
||||
"cursor=bad!",
|
||||
"cursor=e30",
|
||||
"cursor=x&cursor=y",
|
||||
):
|
||||
error(
|
||||
client.get(path + "/messages?" + query, headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
|
||||
def send(i):
|
||||
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
|
||||
assert r.status_code == 201
|
||||
return r.json()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
sent = list(pool.map(send, range(100)))
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute(
|
||||
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
|
||||
)
|
||||
for m in sent:
|
||||
m["created_at"] = "2026-10-17T12:00:00.000000Z"
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
query = {"limit": 17 if cursor else 1}
|
||||
if cursor:
|
||||
query["cursor"] = cursor
|
||||
response = client.get(path, params=query, headers=headers(actors[1]))
|
||||
assert response.status_code == 200
|
||||
page = response.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
assert len(found) == len({x["id"] for x in found}) == 100
|
||||
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
|
||||
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
|
||||
error(
|
||||
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
|
||||
user = actors[0]
|
||||
url = "/api/v1/users/me"
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
repository = client.app.state.repository
|
||||
original = repository.get_user
|
||||
|
||||
def must_be_cached(*args):
|
||||
raise AssertionError("cache hit should avoid fetching immutable user again")
|
||||
|
||||
monkeypatch.setattr(repository, "get_user", must_be_cached)
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
monkeypatch.setattr(repository, "get_user", original)
|
||||
cache = client.app.state.cache.client
|
||||
digest = hashlib.sha256(user["token"].encode()).digest()
|
||||
key = client.app.state.cache.key(digest)
|
||||
assert 0 < cache.ttl(key) <= 3600
|
||||
stale = cache.get(key)
|
||||
cache.flushdb()
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
|
||||
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
|
||||
error(client.get(url, headers=headers(user)), 401, "unauthorized")
|
||||
error(
|
||||
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
|
||||
)
|
||||
|
||||
|
||||
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
|
||||
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
|
||||
database_file.write_text(settings.database_url)
|
||||
cache_file.write_text(settings.cache_url)
|
||||
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
|
||||
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
|
||||
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
|
||||
with TestClient(create_app()) as short:
|
||||
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
|
||||
assert response.status_code == 201
|
||||
token = {"Authorization": "Bearer " + response.json()["token"]}
|
||||
assert short.get("/api/v1/users/me", headers=token).status_code == 200
|
||||
with TestClient(create_app(settings)) as restarted:
|
||||
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
|
||||
time.sleep(2.1)
|
||||
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
|
||||
migrate(settings.database_url, ROOT / "migrations")
|
||||
modified = tmp_path / "001_initial.sql"
|
||||
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
|
||||
with pytest.raises(RuntimeError, match="checksum changed"):
|
||||
migrate(settings.database_url, tmp_path)
|
||||
|
||||
|
||||
def test_logs_and_safe_errors(client, monkeypatch, caplog):
|
||||
canary = "PRIVATE_BODY_CANARY_123"
|
||||
with caplog.at_level("INFO", logger="messenger"):
|
||||
r = client.post(
|
||||
"/api/v1/users",
|
||||
json={"username": "logging", "display_name": canary, "password": PASSWORD},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
assert canary not in caplog.text and PASSWORD not in caplog.text
|
||||
|
||||
def broken(*args):
|
||||
raise RuntimeError(canary)
|
||||
|
||||
monkeypatch.setattr(client.app.state.authentication, "register", broken)
|
||||
response = client.post(
|
||||
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
|
||||
)
|
||||
error(response, 500, "internal_error")
|
||||
assert canary not in response.text and canary not in caplog.text
|
||||
error(
|
||||
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from test_api import error, headers
|
||||
|
||||
from messenger.app import create_app
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
|
||||
def test_message_key_required(client, actors, room, key):
|
||||
h = {"Authorization": "Bearer " + actors[0]["token"]}
|
||||
if key is not None:
|
||||
h["Idempotency-Key"] = key
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
key = "Scope.Key:123"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": key}
|
||||
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
|
||||
assert first.status_code == 201
|
||||
repeat = client.post(
|
||||
path,
|
||||
headers={**h, "Content-Type": "application/json"},
|
||||
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
|
||||
)
|
||||
assert repeat.status_code == 201 and repeat.json() == first.json()
|
||||
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
|
||||
outsider = {**headers(actors[2]), "Idempotency-Key": key}
|
||||
error(
|
||||
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
|
||||
)
|
||||
other_user = {**headers(actors[1]), "Idempotency-Key": key}
|
||||
own = client.post(path, headers=other_user, json={"text": "another author"})
|
||||
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
|
||||
chat = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "another", "member_ids": [actors[0]["id"]]},
|
||||
).json()
|
||||
other_chat = client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
|
||||
)
|
||||
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
|
||||
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
|
||||
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
with TestClient(create_app(settings)) as second:
|
||||
assert second.app.state.instance_id != client.app.state.instance_id
|
||||
|
||||
def send(i):
|
||||
c = client if i % 2 else second
|
||||
r = c.post(path, headers=h, json={"text": "same concurrent body"})
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(send, range(30)))
|
||||
assert len({item[0]["id"] for item in results}) == 1
|
||||
assert len({item[1] for item in results}) == 2
|
||||
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
|
||||
|
||||
|
||||
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
|
||||
repository = client.app.state.repository
|
||||
original = repository._save_request
|
||||
|
||||
def fail_after_message_insert(*args):
|
||||
raise RuntimeError("injected between message and result")
|
||||
|
||||
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
error(
|
||||
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
|
||||
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
|
||||
monkeypatch.setattr(repository, "_save_request", original)
|
||||
result = client.post(path, headers=h, json={"text": "retry after rollback"})
|
||||
assert result.status_code == 201
|
||||
assert (
|
||||
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
|
||||
)
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
first = client.post(path, headers=h, json={"text": "old"}).json()
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute(
|
||||
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
|
||||
"expires_at=clock_timestamp()-interval '1 day'"
|
||||
)
|
||||
second = client.post(path, headers=h, json={"text": "new"})
|
||||
assert second.status_code == 201 and second.json()["id"] != first["id"]
|
||||
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("optimized", [True, False])
|
||||
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
|
||||
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
|
||||
c.get("/api/v1/users/me", headers=headers(actors[0]))
|
||||
pool = c.app.state.repository.pool
|
||||
before = pool.get_stats().get("requests_num", 0)
|
||||
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
|
||||
assert response.status_code == 200 and response.json()["items"] == []
|
||||
assert pool.get_stats()["requests_num"] - before == 3
|
||||
|
||||
|
||||
def test_instance_header_covers_health_errors_and_success(client):
|
||||
values = [
|
||||
client.get("/health/live"),
|
||||
client.get("/health/ready"),
|
||||
client.get("/api/v1/users/me"),
|
||||
client.get("/not-a-route"),
|
||||
]
|
||||
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
|
||||
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
|
||||
@@ -0,0 +1,357 @@
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
from conftest import ROOT
|
||||
from PIL import Image
|
||||
from test_api import error, headers
|
||||
|
||||
from messenger.broker import GROUP, STREAM, Broker, Job
|
||||
from messenger.cleanup import collect
|
||||
from messenger.domain import DomainError
|
||||
from messenger.images import variants
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.relay import publish_one
|
||||
from messenger.worker import handle
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def upload(client, actor, room, data=PNG, media="image/png", name="Фото.png"):
|
||||
return client.post(
|
||||
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
|
||||
headers={**headers(actor), "Content-Type": media},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def broker(settings):
|
||||
b = Broker(settings)
|
||||
assert settings.broker_url.endswith("/1")
|
||||
b.client.flushdb()
|
||||
b.group()
|
||||
yield b
|
||||
b.client.flushdb()
|
||||
b.close()
|
||||
|
||||
|
||||
def process(client, settings, broker):
|
||||
db = client.app.state.repository
|
||||
assert publish_one(db, broker, settings)
|
||||
id, fields = broker.client.xreadgroup(GROUP, "unit", {STREAM: ">"}, count=1)[0][1][0]
|
||||
jsonschema.validate(
|
||||
json.loads(fields["job"]),
|
||||
json.loads((ROOT / "contracts/jobs.schema.json").read_text()),
|
||||
format_checker=jsonschema.FormatChecker(),
|
||||
)
|
||||
handle(db, broker, client.app.state.attachments.store, settings, id, fields["job"])
|
||||
|
||||
|
||||
def test_queued_message_ready_and_all_variants(client, actors, room, settings, broker):
|
||||
r = upload(client, actors[0], room)
|
||||
assert r.status_code == 202 and r.json()["status"] == "queued"
|
||||
a = r.json()
|
||||
assert not a["variants"] and a["sha256"] == hashlib.sha256(PNG).hexdigest()
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
error(client.get(path + "/content", headers=headers(actors[0])), 409, "conflict")
|
||||
key = uuid4().hex
|
||||
msg = client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages",
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={**headers(actors[0]), "Idempotency-Key": key},
|
||||
)
|
||||
assert msg.status_code == 201
|
||||
for suffix in ("", "/content"):
|
||||
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
||||
error(client.get(path + suffix), 401, "unauthorized")
|
||||
process(client, settings, broker)
|
||||
ready = client.get(path, headers=headers(actors[1])).json()
|
||||
assert ready["status"] == "ready" and ready["error_code"] is None
|
||||
assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}
|
||||
assert client.get(path + "/content", headers=headers(actors[1])).content == PNG
|
||||
for v in ready["variants"]:
|
||||
content = client.get(path + "/content?variant=" + v["name"], headers=headers(actors[1]))
|
||||
assert content.status_code == 200 and content.headers["content-type"] == v["content_type"]
|
||||
assert len(content.content) == v["size_bytes"]
|
||||
assert hashlib.sha256(content.content).hexdigest() == v["sha256"]
|
||||
with Image.open(io.BytesIO(content.content)) as image:
|
||||
assert image.size == (v["width"], v["height"])
|
||||
assert not image.getexif()
|
||||
with client.app.state.repository.connection() as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
|
||||
).fetchone()
|
||||
job = Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=a["id"],
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
created_at=a["created_at"],
|
||||
)
|
||||
sid = broker.publish(job)
|
||||
handle(
|
||||
client.app.state.repository,
|
||||
broker,
|
||||
client.app.state.attachments.store,
|
||||
settings,
|
||||
sid,
|
||||
job.model_dump_json(exclude_none=True),
|
||||
)
|
||||
assert client.get(path, headers=headers(actors[0])).json() == ready
|
||||
assert broker.client.xlen(STREAM) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
|
||||
def test_bad_filename(client, actors, room, name):
|
||||
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,media",
|
||||
[
|
||||
(b"", "image/png"),
|
||||
(b"bad", "image/png"),
|
||||
(PNG[:30], "image/png"),
|
||||
(PNG, "image/jpeg"),
|
||||
(PNG, "text/plain"),
|
||||
],
|
||||
)
|
||||
def test_bad_image(client, actors, room, data, media):
|
||||
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
|
||||
|
||||
|
||||
def test_upload_limits_and_pixel_bomb(client, actors, room):
|
||||
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
|
||||
error(
|
||||
upload(client, actors[0], room, (b"x" * 1048576 for _ in range(11))),
|
||||
413,
|
||||
"payload_too_large",
|
||||
)
|
||||
data = bytearray(PNG)
|
||||
data[16:24] = struct.pack(">II", 10000, 10000)
|
||||
data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xFFFFFFFF)
|
||||
error(upload(client, actors[0], room, bytes(data)), 413, "payload_too_large")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [(320, 200), (17, 11), (1, 400), (400, 1), (1281, 713)])
|
||||
def test_transform_dimensions_alpha_and_exif(size):
|
||||
raw = io.BytesIO()
|
||||
Image.new("RGBA", size, (1, 2, 3, 0)).save(raw, "PNG")
|
||||
results = {name: (data, dims, media) for name, data, dims, media in variants(raw, 1)}
|
||||
scale = max(size)
|
||||
assert results["thumbnail"][1] == tuple(max(1, n * min(256, scale) // scale) for n in size)
|
||||
assert results["crop"][1] == (128, 128)
|
||||
assert max(results["optimized"][1]) <= 1280
|
||||
with Image.open(io.BytesIO(results["thumbnail"][0])) as image:
|
||||
assert image.getpixel((0, 0)) == (255, 255, 255)
|
||||
|
||||
|
||||
def test_exif_orientation_and_original_unchanged():
|
||||
raw = io.BytesIO()
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
Image.new("RGB", (320, 200), "red").save(raw, "JPEG", exif=exif)
|
||||
before = raw.getvalue()
|
||||
result = list(variants(raw, 1))
|
||||
assert result[0][2] == (160, 256) and raw.getvalue() == before
|
||||
for _, data, _, _ in result:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
assert not image.getexif()
|
||||
|
||||
|
||||
def test_failed_retry_replay_and_ready_replay_rejected(
|
||||
client, actors, room, settings, broker, monkeypatch
|
||||
):
|
||||
a = upload(client, actors[0], room).json()
|
||||
store, db = client.app.state.attachments.store, client.app.state.repository
|
||||
original = store.put
|
||||
monkeypatch.setattr(
|
||||
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
|
||||
)
|
||||
process(client, replace(settings, job_max_attempts=1), broker)
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
assert client.get(path, headers=headers(actors[0])).json()["error_code"] == "retry_exhausted"
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages",
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
|
||||
),
|
||||
409,
|
||||
"conflict",
|
||||
)
|
||||
Pipeline(db, settings).replay(a["id"])
|
||||
monkeypatch.setattr(store, "put", original)
|
||||
process(client, settings, broker)
|
||||
assert client.get(path, headers=headers(actors[0])).json()["status"] == "ready"
|
||||
with pytest.raises(ValueError):
|
||||
Pipeline(db, settings).replay(a["id"])
|
||||
|
||||
|
||||
def test_invalid_envelope_quarantined(client, settings, broker):
|
||||
sid = broker.client.xadd(STREAM, {"job": '{"url":"http://untrusted.invalid/"}'})
|
||||
handle(
|
||||
client.app.state.repository,
|
||||
broker,
|
||||
client.app.state.attachments.store,
|
||||
settings,
|
||||
sid,
|
||||
'{"url":"http://untrusted.invalid/"}',
|
||||
)
|
||||
assert broker.client.xlen("invalid-jobs") == 1
|
||||
assert "url" not in json.dumps(broker.client.xrange("invalid-jobs"))
|
||||
|
||||
|
||||
def test_staging_failure_has_no_accepted_job(client, actors, room, monkeypatch):
|
||||
store = client.app.state.attachments.store
|
||||
monkeypatch.setattr(
|
||||
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
|
||||
)
|
||||
error(upload(client, actors[0], room), 503, "unavailable")
|
||||
with client.app.state.repository.connection() as conn:
|
||||
assert conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] == 0
|
||||
|
||||
|
||||
def test_gc_minimum_retention(client):
|
||||
with pytest.raises(ValueError):
|
||||
collect(client.app.state.repository, client.app.state.attachments.store, 0, True)
|
||||
|
||||
|
||||
def test_gc_fences_paused_upload_before_job_commit(client, actors, room, settings, monkeypatch):
|
||||
db, store = client.app.state.repository, client.app.state.attachments.store
|
||||
p = Pipeline(db, settings)
|
||||
id = str(uuid4())
|
||||
key = "staging/" + id
|
||||
p.reserve(id, key)
|
||||
store.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
|
||||
# Simulated S3 object age; actual private test-bucket deletion and DB locks are real.
|
||||
old = datetime.now(UTC) - timedelta(seconds=301)
|
||||
head = store.head(key)
|
||||
head["LastModified"] = old
|
||||
objects = [{"Key": key, "ETag": head["ETag"], "Size": len(PNG), "LastModified": old}]
|
||||
monkeypatch.setattr(store, "head", lambda _: head)
|
||||
monkeypatch.setattr(
|
||||
store.client,
|
||||
"get_paginator",
|
||||
lambda _: SimpleNamespace(paginate=lambda **_: [{"Contents": objects}]),
|
||||
)
|
||||
assert collect(db, store, 300, True)["protected"] == 1
|
||||
with db.connection(write=True) as conn:
|
||||
conn.execute(
|
||||
"UPDATE upload_reservations SET expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE id=%s",
|
||||
(id,),
|
||||
)
|
||||
assert collect(db, store, 300, False)["eligible"] == 1
|
||||
assert collect(db, store, 300, True)["deleted"] == 1
|
||||
with pytest.raises(DomainError) as expired:
|
||||
p.accept(
|
||||
id,
|
||||
room["id"],
|
||||
actors[0]["id"],
|
||||
"paused.png",
|
||||
"image/png",
|
||||
len(PNG),
|
||||
hashlib.sha256(PNG).hexdigest(),
|
||||
key,
|
||||
)
|
||||
assert expired.value.status == 503 and db.attachment(id) is None
|
||||
|
||||
|
||||
def test_version_fence_rejects_old_writer(client, actors, room, settings, broker):
|
||||
a = upload(client, actors[0], room).json()
|
||||
db, store = client.app.state.repository, client.app.state.attachments.store
|
||||
with db.connection() as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
|
||||
).fetchone()
|
||||
envelope = Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=a["id"],
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
created_at=a["created_at"],
|
||||
)
|
||||
pipeline = Pipeline(db, settings)
|
||||
outcome, old = pipeline.claim(envelope)
|
||||
assert outcome == "claimed"
|
||||
pipeline.replay(a["id"], new_version=True)
|
||||
process(client, settings, broker)
|
||||
current = db.attachment(a["id"])
|
||||
assert current["pipeline_version"] == 2 and current["storage_state"] == "ready"
|
||||
assert not pipeline.publish(old, "objects/old-result", [])
|
||||
handle(
|
||||
db,
|
||||
broker,
|
||||
store,
|
||||
settings,
|
||||
broker.publish(envelope),
|
||||
envelope.model_dump_json(exclude_none=True),
|
||||
)
|
||||
assert db.attachment(a["id"])["object_key"] == current["object_key"]
|
||||
|
||||
|
||||
def test_foreign_chat_attachment_is_not_linkable(client, actors, room):
|
||||
a = upload(client, actors[0], room).json()
|
||||
other = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "other", "member_ids": [actors[0]["id"]]},
|
||||
).json()
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
),
|
||||
404,
|
||||
"not_found",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patch",
|
||||
[
|
||||
{"schema_version": True},
|
||||
{"traceparent": None},
|
||||
{"tracestate": None},
|
||||
{"url": "https://untrusted.invalid/"},
|
||||
{"job_id": "not-a-uuid"},
|
||||
{"pipeline_version": 0},
|
||||
{"attempt": 0},
|
||||
{"created_at": "2026-10-17T12:00:00"},
|
||||
{"traceparent": "invalid"},
|
||||
{"type": "download.anything"},
|
||||
],
|
||||
)
|
||||
def test_job_schema_rejects_invalid_carrier(client, settings, broker, patch):
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"type": "attachment.process.v1",
|
||||
"job_id": str(uuid4()),
|
||||
"attachment_id": str(uuid4()),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-10-17T12:00:00Z",
|
||||
**patch,
|
||||
}
|
||||
data = json.dumps(payload)
|
||||
sid = broker.client.xadd(STREAM, {"job": data})
|
||||
handle(
|
||||
client.app.state.repository, broker, client.app.state.attachments.store, settings, sid, data
|
||||
)
|
||||
assert broker.client.xrange("invalid-jobs")[0][1]["reason"] == "invalid_envelope"
|
||||
Reference in New Issue
Block a user