5 Commits
74 changed files with 12494 additions and 39 deletions
+6
View File
@@ -17,4 +17,10 @@ backups
__pycache__
node_modules
evidence
student-tests
tests
research
REPORT.md
SCHEMA.md
requirements-dev.txt
.DS_Store
+5 -1
View File
@@ -21,5 +21,9 @@ MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=20
STAGING_RETENTION_SECONDS=86400
PUBLIC_PORT=8085
DATABASE_POOL_SIZE=8
DATABASE_READ_AUTOCOMMIT=1
+12 -14
View File
@@ -1,18 +1,16 @@
## Лабораторная 5
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Горбанюк / БВТ2301.
Целевая оценка: 5. Трек: 2 — идемпотентность versioned pipeline.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Реализация и воспроизводимая последовательность запуска: [REPORT.md](../REPORT.md).
Исходники приложения — src/, собственные проверки — student-tests/, результаты — evidence/.
Сохранено 84 собственных проверочных случаев; условия и результаты конкретных запусков приведены в отчёте.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Контракт, исходное задание и публичные тесты сохранены. Секреты создаются локально и не входят в Git.
Ограничения и использованные первичные источники перечислены в REPORT.md.
- [x] Реализован обязательный минимум и заявленный трек.
- [x] Приведены команды запуска, проверок и безопасной остановки.
- [x] Сохранены результаты проверок и ограничения выводов.
- [x] Credentials и приватные ключи находятся вне репозитория.
+23
View File
@@ -0,0 +1,23 @@
name: Application integration
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.14.7'
- run: python -m pip install uv==0.12.13
- run: uv sync --locked
- run: make bootstrap lint check BASE_URL=http://localhost:8085
- run: VCS_REF=$(git rev-parse HEAD) docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
- run: PYTEST_ADDOPTS=--disable-warnings make unit
- run: make test BASE_URL=http://localhost:8085
- run: uv run --locked python scripts/verify_cluster.py --base-url http://localhost:8085 --output /tmp/cluster-ci.json
- run: uv run python scripts/verify_pipeline.py --base-url http://localhost:8085 --output /tmp/pipeline-ci.json
- if: always()
run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml down
+1
View File
@@ -0,0 +1 @@
3.14.7
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
WORKDIR /app
RUN python -m pip install --no-cache-dir uv==0.12.13
ENV UV_PYTHON_DOWNLOADS=never
COPY pyproject.toml uv.lock .python-version ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
COPY src ./src
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY migrations /app/migrations
ARG VCS_REF=uncommitted
LABEL org.opencontainers.image.revision=$VCS_REF
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
APP_LOG_LEVEL=INFO
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
CMD ["python", "-m", "messenger"]
+19 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8085
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,21 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down unit lint verify-cluster verify-pipeline
bootstrap:
$(PYTHON) scripts/bootstrap.py
up:
docker compose --env-file .env up -d --build --wait --wait-timeout 180
down:
docker compose --env-file .env stop --timeout 7
unit:
uv run pytest --tb=short
lint:
uv run ruff check .
uv run ruff format --check .
verify-cluster:
uv run python scripts/verify_cluster.py --base-url "$(BASE_URL)"
verify-pipeline:
uv run python scripts/verify_pipeline.py --base-url "$(BASE_URL)"
+198 -23
View File
@@ -1,37 +1,212 @@
# Отчёт — лабораторная 5
# Лабораторная 5 — очереди и обработка изображений
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
## Запуск из чистого clone
**Горбанюк, БВТ2301. Целевая оценка: 5, трек 2 — идемпотентность versioned pipeline с внедрением отказов.**
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Решение опубликовано в публичном форке исходного проекта; МР преподавателю открыт. Ветка `solution/bvt2301-gorbanyuk`, проверенная версия — тег `lab5-verified`. Название МР: **«ЛР 5 — БВТ2301 — Горбанюк — на 5»**. README задания, COURSE, CONTRIBUTING, контракты и публичные тесты сохранены.
## Архитектура и решения
Публичный форк: <https://gitea.privcloud.tech/Wemiliam/mtusi2026lab5>. Ветка решения: `solution/bvt2301-gorbanyuk`, целевая ветка преподавателя — `main`. Ветка опубликована; МР преподавателю: <https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/10>.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
## Запуск из самостоятельного clone
Нужны Git, Docker с Compose v2+, Python **3.14.7**, `uv 0.12.13`, доступ к реестрам и официальным Go-модулям при первом build. Проверено на ARM64, Docker 29.8.1 / Compose 5.5.1; Docker VM: 18 CPU, 7.75 GiB RAM. Выделите Docker около 6 GiB RAM и несколько GiB диска. Соседние лабораторные не нужны.
Команды выполняются из корня клонированного репозитория.
```sh
uv sync --locked
make bootstrap
VCS_REF=$(git rev-parse HEAD) make up
make check validate PYTHON=.venv/bin/python
make test PYTHON=.venv/bin/python
```
API: `http://127.0.0.1:8085`, две реплики за HAProxy. У API нет собственных опубликованных портов. `bootstrap` создаёт случайные локальные секреты в игнорируемом `secrets/` и `.env`, сохраняя прежние значения. Каталог 0700; файлы читаются контейнерным UID через Compose secrets. Это локальные файлы, не централизованное зашифрованное хранилище секретов.
Одноразовый `migrate` применяет SQL под advisory lock, проверяя checksum ранее применённых миграций. `bucket-init` создаёт private buckets и IAM identities. API ждёт их успешного завершения. MinIO и mc собираются из официальных исходников, закреплённых на `RELEASE.2025-10-15T17-29-55Z` / `RELEASE.2025-08-13T08-35-41Z`, Go 1.26.7. Первый холодный build может занять несколько минут. Бинарник имеет label `DEVELOPMENT.GOGET`; фактическая версия исходников проверяется через Go build info, а не этот label. Это учебный pin, без заявления о текущей production-поддержке.
Для собственных интеграционных проверок включите локальные test ports и отдельный тестовый bucket:
```sh
docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
make lint unit
uv run python scripts/verify_pipeline.py --output /tmp/pipeline-new.json
uv run python scripts/verify_cluster.py --output /tmp/cluster-new.json
```
Override открывает **только loopback**: PG 55437, кэш 56384, брокер 56385, S3 59005. Он включает операторские single-use fault hooks и timeout job **5 s / lease 8 s**. Обычный Compose оставляет hooks выключенными, timeout **20 s / lease 23 s**. Не запускайте одновременно сценарии, меняющие состояние одного стенда. Выходной файл должен быть новым, предыдущие доказательства не перезаписываются.
Реальная проверка возраста и очистки:
```sh
uv run python scripts/verify_cleanup.py seed --seed /tmp/gc-seed-new.json
# Пока объекты стареют, выполните другие проверки.
uv run python scripts/verify_cleanup.py verify --seed /tmp/gc-seed-new.json --output /tmp/gc-new.json
```
Второй вызов дожидается >=301 s после terminal failed. Возраст S3 и часы не подменяются. Seed содержит только ID, без credentials. Скрипт проверяет orphan, failed retention, защиту active и опубликованных объектов; после проверки отложенная работа освобождается и доходит до ready. Быстрый unit-тест гонки с API моделирует возраст метаданных отдельно, не заменяя эту проверку.
```sh
make down # Остановить, сохранив данные.
docker compose --env-file .env up -d --wait # Возобновить.
# Только намеренное удаление ВСЕХ учебных данных этого проекта:
docker compose --env-file .env down --volumes
```
Проект `mtusi-lab5` имеет собственные `postgres_data`, `s3_data`, `broker_data`. Кэш одноразовый: сессии хранятся в PostgreSQL. Секреты/volumes не входят в Git или bundle.
## Архитектура и транзакционные границы
```mermaid
flowchart LR
C[Клиент] --> LB[HAProxy]
LB --> A1[API 1]
LB --> A2[API 2]
A1 --> PG[(PostgreSQL)]
A2 --> PG
A1 --> S[(Приватный S3)]
A2 --> S
PG --> R[Outbox relay и watchdog]
R --> B[(Valkey Streams AOF)]
B --> W[Worker supervisor]
W --> P[Ограниченные дочерние процессы]
P --> S
P --> PG
```
Перенесены собственные FastAPI API, Argon2id, серверные Bearer-сессии, keyset pagination, членство в чате, PostgreSQL, кэш, две реплики и атомарная идемпотентность сообщений. Из ЛР4 сохранены приватность, ограничения файлов, оригинальные байты и связь с чатом. Объявленный переход: **201 ready → 202 queued**; queued/processing можно прикреплять к сообщению, failed нельзя.
БД — источник истины о состоянии, S3 — о байтах. Брокер — **отдельный от кэша** Valkey 8.1.3: persistent volume, AOF `appendfsync always`, RDB `60 1`, максимум 192 MiB данных, `noeviction`. Кэш — другой процесс без persistence, `allkeys-lru`. Переполнение/остановка брокера задерживает relay; PG outbox остаётся сохранённым.
1. API проверяет сессию/чат, имя, MIME, фактический поток <=10 MiB, время приёма <=15 s, целостность PNG/JPEG, <=40 млн пикселей. Pixel guard действует **до** полной распаковки. На API один upload slot, ожидание <=5 s, затем 429. Временный файл в tmpfs не является durable staging.
2. БД создаёт upload reservation с lease 120 s. Проверенный оригинал записывается в persistent private S3 `staging/<UUID>`. Имя клиента не используется как ключ. PUT имеет ограниченные socket timeouts без автоматических повторов.
3. **Одна PG-транзакция** блокирует/проверяет reservation, создаёт attachment=queued, job и outbox, отмечает handoff. После COMMIT API возвращает **202 со снимком queued**. Thumbnail ещё не обещан; worker может успеть завершить до первого GET.
4. Relay выбирает due outbox через `FOR UPDATE SKIP LOCKED`, делает XADD, затем фиксирует published_at. Потеря ответа XADD/COMMIT вызывает повтор с тем же job_id. До шага 3 возможен непринятый orphan: общей транзакции S3+PG нет. Reservation и retention GC закрывают это окно без ложного 202.
5. Worker получает COUNT=1 из consumer group. В короткой PG-транзакции захватывает job, увеличивает attempt, сохраняет fence/lease. Pillow работает вне SQL-транзакции, в **отдельном OS-процессе**. Supervisor держит не более двух child, убивает зависший по wall clock. Child RLIMIT_AS 768 MiB; worker container 1536 MiB / 2 CPU / tmpfs 64 MiB / pids 96.
6. Worker сверяет исходный размер/SHA-256, повторно валидирует изображение, сохраняет original и три варианта в `objects/<attachment>/v<version>/<variant>`. После всех PUT атомарно публикует метаданные **при совпадении текущей версии, fence и действующего lease**.
7. XACK означает устойчивый эффект. XACK+XDEL атомарны в Lua для единственной consumer group; pending не обрезаются через MAXLEN. XAUTOCLAIM восстанавливает неподтверждённые записи. При временной ошибке ACK выполняется после durable retry-intent, при failed — после terminal registry.
API readiness проверяет БД/миграции и S3; кэш, broker, relay и worker не входят в обязательный синхронный путь. При остановленном worker API способен устойчиво принять файл, но обработка задержится. Liveness проверяет только процесс. `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE`, S3 file credentials соответствуют `.env.example`; pool size=8, upload=10 MiB/40M px, concurrency=2, max attempts=5, staging retention=86400 s. Secret-поля Settings исключены из repr.
## Состояния, retry, версии и идемпотентность
`queued → processing → ready`; временная ошибка/истёкший lease возвращают queued с due time. Постоянная ошибка или пять неудач дают failed с безопасным error_code. **Attempt — управляемая попытка обработки, не число доставок брокера.** Envelope строго соответствует `contracts/jobs.schema.json`: schema_version/type/job_id/attachment_id/pipeline_version/attempt/created_at; trace context зарезервирован схемой. Bytes, credentials, filename, URL в stream не передаются. Недоверенный envelope проверяется; неверный помещается в служебный DLQ без копирования произвольных полей.
`attachment_jobs WHERE state='failed'` — durable failed registry. Audit фиксирует claim, публикации, отказы, replay, старые доставки, fence. Temporary storage/network ошибки повторяются; повреждение/неверный формат/пиксели/MemoryError постоянны. Supervisor timeout восстанавливает watchdog relay. Full jitter: равномерно от **0.25 s до min(30, 2^attempt)**, абсолютное время due сохраняется в БД. После пятой неудачи автоматических повторов нет.
```sh
docker compose --env-file .env exec worker python -m messenger.operator failed
docker compose --env-file .env exec worker python -m messenger.operator replay <attachment-uuid>
docker compose --env-file .env exec worker python -m messenger.operator new-version <attachment-uuid>
docker compose --env-file .env run --rm reconciler
docker compose --env-file .env run --rm reconciler python -m messenger.cleanup --apply
```
CLI доступен только оператору локального стенда, не публичному HTTP клиенту. Replay разрешён только для failed с retained source: CLI проверяет наличие объекта, сохраняет job_id/версию, начинает новый ограниченный цикл попыток с audit. Ready replay отклоняется. New-version реализует **v1→v2**: optimized WebP quality 82→76; другие преобразования одинаковы. Source key фиксирован в job; для reprocess готового attachment берётся опубликованный original.
Ключ эффекта **(attachment_id, pipeline_version, variant)**. UNIQUE(attachment_id,pipeline_version) защищает logical job. Имена объектов детерминированы; повторные PUT занимают те же ключи, без второго набора вариантов. Старый процесс может дописать свой prefix v1, но не v2. PG fence/lease дополнительно запрещают устаревший COMMIT. Ready-дубль сверяет полноту и HEAD size/checksum, затем ACK без resize. Старые version/attempt не захватывают новую работу. **Менять алгоритм, сохраняя номер версии, нельзя:** это нарушит детерминированность.
## Изображения, приватность и очистка
| Вариант | Параметры |
| --- | --- |
| original | Побайтная копия, тот же MIME/размер/SHA-256, включая исходные EXIF |
| thumbnail | JPEG quality 85/subsampling 0, LANCZOS, <=256×256, без увеличения |
| crop | JPEG quality 85/subsampling 0, центральный квадрат 128×128, увеличение маленьких допустимо |
| optimized | WebP quality 82(v1)/76(v2), method 4, <=1280×1280 без увеличения |
EXIF orientation применяется до преобразований; alpha компонуется на белый. Размеры вписывания округляются вниз, минимум 1 px: 320×200 → 256×160. Generated variants не содержат EXIF. WebP не обязан быть меньше каждого маленького исходника.
Metadata и каждый download требуют сессию/членство; чужое — 404, content queued/processing/failed — 409. Download идёт через API, без redirect, со своим MIME/размером, private/no-store/nosniff. Storage keys отсутствуют в публичных метаданных. S3 anonymous GET — 403. IAM разделён: API пишет staging/читает final; worker читает оба prefix/пишет final; cleanup читает/удаляет, не пишет; тестовая identity ограничена отдельным bucket. Root keys не доступны API/worker.
Cleanup по умолчанию dry-run, применение явное. Retention **86400 s**, hard minimum 300 s. Удаляются старые непринятые orphan, staging после опубликованного результата, остатки старых версий. Active queued/processing защищает объекты attachment. Текущие published originals/variants сохраняются даже без message-link. Failed source сохраняется >=retention после отказа, затем удаляется с source_expired/audit; replay потерянного источника не обещается.
Перед DELETE повторяется HEAD ETag/size/age; недоступность не считается 404. Коллекторы сериализованы advisory lock. Replay/new-version и GC согласуются row lock attachment; reservation защищает ещё не завершённый handoff API: expired/abandoned upload не сможет после удаления вернуть 202.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Команда/сценарий | Наблюдение и файл |
| --- | --- | --- |
| Контракт и исходные tests | `make check validate`, Git compare | PASS, неизменённый skeleton; `evidence/contract.txt`, `audit.json` |
| 202, реальный thumbnail, сессии, ACL | `make test` | PASS; `evidence/smoke.txt` |
| API, гонки, размеры/EXIF/alpha, replay, GC fence | `make unit` | 84 проверки; `evidence/unit.txt` |
| Worker down → queued message → API restart → broker SIGKILL → ready | Первый сценарий `verify_pipeline.py` | Реальные CID/StartedAt, сохранённые Stream IDs; `evidence/pipeline.json` |
| COMMIT/publish gap, publish до outbox COMMIT | Первые два сценария матрицы | Восстановление без ручной новой job; `pipeline.json` |
| Crash до/после S3 и после result COMMIT до ACK | Три сценария worker | Attempts 2/2/1, один набор результатов; `pipeline.json` |
| Transient/permanent, jitter, timeout, exhaustion, replay | Четыре сценария матрицы | Safe failed, lease recovery, audited CLI replay; `pipeline.json` |
| Bounded concurrency/time/RAM, варианты | Supervisor + checksum/decode/cgroup | Три реальных варианта; `pipeline.json` |
| Versioned pipeline, reorder, stale writer | v2 перед v1; 44 доставки и publication fence | Новая версия/байты не перезаписаны; `pipeline.json` |
| Private S3, чтение другой API | Anonymous/IAM отрицательные checks; actual uploader stopped | 403 и download с другой реплики; `pipeline.json` |
| Retention/cleanup | `verify_cleanup.py`, реальный возраст >=301 s | Active/ready сохранены, expired/orphan удалены; `cleanup.json` |
| Failover/rolling/persistence | `verify_cluster.py`, публичные resilience hooks | Настоящие процессы/stores; `cluster.json`, `failover.txt`, `persistence.txt` |
| Чистый clone | Новые secrets/volumes и отдельный Compose project | `clean-clone.json`, `clean-clone.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Исследование трека 2
## Эксперименты и отказы
Гипотеза: at-least-once + детерминированные versioned keys + PG fence сохраняют подтверждённые работы и запрещают старому writer опубликоваться поверх нового. `verify_pipeline.py` воспроизводит 10 сценариев на настоящих PG/S3/Stream/processes. Hooks одноразовые, операторские, выключены в обычном Compose. Crash — `os._exit(87)` worker child/relay; hang убивает supervisor. After_effect расположен после PUT original, до полного результата; after_commit — после всех вариантов/COMMIT до ACK.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Raw JSON содержит время, attempts, audit/PID, object count, версии, CID/StartedAt, cgroup memory/CPU. Длительности включают остановки/ожидание redelivery, **не являются p95 обычного upload**. `memory.peak` — пик всей cgroup с дочерними процессами, не RSS одного процесса. Реальные варианты декодируются и проверяются по SHA-256. Инварианты проверяются assertions. Матрица перегрузки/concurrency не заявляется: выбран трек 2.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги
### Измеренные результаты локального прогона
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
| Сценарий | Время, s | Attempts последнего цикла |
| --- | ---: | ---: |
| commit_publish_gap_api_restart_broker_sigkill | 12.72 | 1 |
| publish_before_outbox_commit_duplicate | 14.33 | 1 |
| worker_crash_before_effect | 13.89 | 2 |
| worker_crash_after_effect | 13.86 | 2 |
| worker_crash_after_commit | 11.78 | 1 |
| transient_2 | 8.71 | 3 |
| hang_1 | 13.66 | 2 |
| permanent_1_failed_then_explicit_replay | 6.04 | 1 |
| transient_5_failed_then_explicit_replay | 30.80 | 1 |
| reorder_stale_v1_after_v2_and_publication_fence | 19.35 | 1 |
Матрица: 11 подтверждённых upload, потеряно 0, устаревших перезаписей 0; по 4 объекта текущей версии. У failed перед replay отдельно зафиксированы исходные attempts 1/5; число в таблице для них относится к успешному новому циклу. Пик worker cgroup 209.18 MiB; OOM kills=0. Измерение относится к небольшому fixture 320×200 и этому прогону.
Реальный GC: failed age 301.00 s; dry-run нашёл 14 кандидатов и ничего не удалил; apply удалил 14. Активный staging сохранён и затем обработан; 4 текущих published objects остались неизменными. Эти counts зависят от числа прошлых синтетических загрузок, критерии проверяются по конкретным seed ID.
Отказы API под шестью потоками: реплика 1: 201 подтверждённых записей, первое успешное чтение/запись после начала действия через 1.91 s, ошибки 2.92%; реплика 2: 165 подтверждённых записей, первое успешное чтение/запись после начала действия через 1.86 s, ошибки 6.37%. Rolling replacement: 563 подтверждённых записей, все сохранены, ошибки 5.13%. Не обещается отсутствие транзиентных 5xx во время отказа; сообщения повторяются тем же Idempotency-Key. Публичный failover отдельно: 130 успешных записей, 2 неудачные попытки, первое восстановление 0.984 s. Persistence перезапустил реальные API/PG/S3/broker/cache/worker/relay/LB и сохранил действительную сессию. Эти показатели относятся к локальной синтетической проверке отказов, не к оценке пропускной способности очереди.
## Ответы на вопросы README
**Publisher confirm и consumer ack.** Confirm подтверждает приём от producer брокером; он не говорит, что worker сделал картинку. ACK подтверждает устойчивую обработку либо сохранённое решение о retry/failed. В RabbitMQ это разные механизмы. Здесь publisher boundary — ответ XADD при AOF appendfsync always; consumer boundary — XACK. Неопределённый исход publish приводит к повторной отправке; смерть после result COMMIT до XACK — к повторной доставке.
**Что сохранено при 202?** Все байты уже приняты и проверены, original находится в persistent staging, PG COMMIT содержит queued attachment, logical job и outbox-intent. Thumbnail ещё не обещан. Брокер может быть недоступен: relay отправит позже. Локальная память/tmpfile не входят в гарантию. До 202 неоднозначный отказ может оставить orphan; повтор upload создаёт новый ID, поскольку идемпотентность upload контрактом не задана.
**Почему нормальны повторы; где ключ?** Нельзя атомарно объединить PG COMMIT, S3 PUT и ACK разных систем. Всегда есть окно «эффект выполнен, подтверждение потеряно». Job_id стабилен при redelivery, UNIQUE защищает logical job, (attachment_id,pipeline_version,variant) — эффект, fence — право публикации. Это **не exactly-once delivery**: доставок/PUT может быть несколько; один текущий набор метаданных — свойство нашего идемпотентного эффекта в заявленной модели.
**Как отличить зависший worker от долгого?** По одному молчанию достоверно невозможно. Мы задаём бюджет и lease: до истечения захват действителен, после разрешено восстановление. Supervisor убивает child по timeout; relay после более длинного lease создаёт bounded retry. Легитимная долгая обработка тоже может быть прервана — цена конечного бюджета. Новая попытка получает новый fence; старая не может COMMIT. Количество процессов/prefetch ограничено; XAUTOCLAIM восстанавливает доставку, а PG lease определяет право на эффект.
Подходы: разделение HTTP/domain/storage; явные транзакции; transactional outbox; at-least-once consumer group; идемпотентный consumer; конечный автомат; fencing/leases; bounded retry/backoff/jitter; failed registry; versioned namespace; least privilege; fault injection и проверяемые инварианты. Очередь сама по себе этих свойств не даёт.
## Ограничения
Один host/PG/S3/broker не переживут потерю всех дисков/VM. Backup, репликация stores и удалённый TLS не реализованы. HTTP — только loopback/синтетические данные. Остановка relay задерживает также lease recovery, сохраняя задания. Оператор наблюдает failed/backlog/disk и запускает cleanup; scheduler не добавлен. Восстановление утраченных published objects из backup не реализовано. Audit/jobs не архивируются автоматически. PG/S3 место конечно; admission по размеру backlog не исследовался.
Тесты — воспроизводимая проверка перечисленных окон, а не математическое доказательство всех interleavings. Проверены два worker slots и версии 1/2; большие картинки под максимальной конкуренцией отдельно не исследованы. Конечные RAM/time лимиты могут отклонить легитимный большой файл; OOM/timeout восстанавливаются по lease с конечным бюджетом попыток. SHA-256 проверяет целостность, не авторство. Новая версия алгоритма требует реализации/тестирования, не только изменения числа.
## Источники и библиотеки
Python 3.14.7; FastAPI 0.143.3/Pydantic 2.14.1/Uvicorn 0.54.0; psycopg 3.3.6, PostgreSQL 18.3; redis client 8.1.0, Valkey 8.1.3; boto3 1.43.109, Pillow 13.0.0; HAProxy 3.2.1. Python-зависимости закреплены `uv.lock`, images/source pins — Compose/Dockerfile/infra.
- Нормативные README, contracts/README.md, attachment-states.md, jobs.schema.json этого skeleton.
- [Valkey XREADGROUP](https://valkey.io/commands/xreadgroup/), [XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [XACK](https://valkey.io/commands/xack/), [persistence](https://valkey.io/topics/persistence/).
- [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acknowledgements](https://www.rabbitmq.com/docs/confirms) — теория границ, RabbitMQ в стенде не используется.
- [PostgreSQL row locks](https://www.postgresql.org/docs/18/explicit-locking.html), [SKIP LOCKED](https://www.postgresql.org/docs/18/sql-select.html).
- [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [file formats](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html).
- [MinIO source pin](https://github.com/minio/minio/tree/RELEASE.2025-10-15T17-29-55Z), [mc source pin](https://github.com/minio/mc/tree/RELEASE.2025-08-13T08-35-41Z).
Заявленные результаты получены запуском сохранённых проверок; ограничения и операторские вмешательства описаны выше.
## Финальная проверка самостоятельности
Отдельный clone исходного commit `25e4ff9` с новыми secrets и пустыми volumes прошёл сборку, lint/format, offline/full contract checks, публичный smoke, все 84 собственных теста, полную матрицу из 10 сценариев и публичный persistence с той же сессией. Реальные StartedAt изменились у API, PG, S3, брокера, кэша, worker, relay и LB. SHA-256 каждого установленного Python-модуля совпал с исходниками на обеих API-репликах; OCI revision соответствует проверенному commit. Порты API не опубликованы. Raw второй матрицы — `evidence/clean-pipeline.json`.
После проверки временные контейнеры/volumes и clone удалены. Основной стенд остановлен, его данные сохранены. Финальный commit добавляет только отчёт/доказательства; исходники соответствуют проверенному `25e4ff9`. ЛР1–ЛР4 остались на прежних чистых версиях.
## Соответствие ревизий
После редактуры отчёта, шаблона и служебных метаданных сохранено соответствие прежних и текущих Git-ревизий: [revision-map.json](evidence/revision-map.json). Код приложения, миграции, зависимости, Dockerfile, конфигурация сервисов и исходные публичные проверки не изменены. Численные результаты и исторические image/source identifiers в evidence сохранены в исходном виде; их не следует трактовать как новый запуск измерений.
+21
View File
@@ -0,0 +1,21 @@
# Only DB/cache test ports; API replicas always stay behind the LB.
services:
postgres:
ports: ["127.0.0.1:${TEST_PG_PORT:-55437}:5432"]
valkey:
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56384}:6379"]
s3:
ports: ["127.0.0.1:${TEST_S3_PORT:-59005}:9000"]
bucket-init:
environment:
CREATE_TEST_BUCKET: '1'
broker:
ports: ["127.0.0.1:${TEST_BROKER_PORT:-56385}:6379"]
worker:
environment:
ENABLE_FAULT_INJECTION: '1'
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
relay:
environment:
ENABLE_FAULT_INJECTION: '1'
JOB_TIMEOUT_SECONDS: ${TEST_JOB_TIMEOUT_SECONDS:-5}
+255
View File
@@ -0,0 +1,255 @@
name: mtusi-lab5
x-api: &api
image: ${API_IMAGE:-mtusi-messenger:lab5}
build:
context: .
args:
VCS_REF: ${VCS_REF:-uncommitted}
user: "10001:10001"
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
pids_limit: 96
cpus: ${API_CPU:-1}
mem_limit: ${API_MEMORY:-512m}
memswap_limit: ${API_MEMORY:-512m}
stop_grace_period: 7s
secrets:
- database_url
- cache_url
- source: s3_api_access_key_id
target: s3_access_key_id
- source: s3_api_secret_access_key
target: s3_secret_access_key
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
APP_ENV: lab5
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
networks: [backend]
services:
postgres:
image: postgres:18.3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql]
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
-c, shared_preload_libraries=pg_stat_statements]
healthcheck:
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
cpus: ${POSTGRES_CPU:-1}
mem_limit: 768m
memswap_limit: 768m
networks: [backend]
valkey:
image: valkey/valkey:8.1.3-alpine
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
--maxmemory-policy, allkeys-lru]
healthcheck:
test: [CMD, valkey-cli, ping]
interval: 2s
timeout: 1s
retries: 20
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
networks: [backend]
migrate:
<<: *api
command: [python, -m, messenger.migrate]
secrets: [database_url]
healthcheck: {disable: true}
depends_on:
postgres: {condition: service_healthy}
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
valkey: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
lb:
image: haproxy:3.2.1-alpine
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports: ["127.0.0.1:${PUBLIC_PORT:-8085}:8080"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 2s
retries: 30
depends_on:
api: {condition: service_healthy}
networks: [backend]
broker:
image: valkey/valkey:8.1.3-alpine
command: [valkey-server, --appendonly, "yes", --appendfsync, always, --save, "60 1",
--maxmemory, 192mb, --maxmemory-policy, noeviction]
volumes: [broker_data:/data]
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
healthcheck:
test: [CMD, valkey-cli, ping]
interval: 2s
timeout: 1s
retries: 30
networks: [backend]
relay:
<<: *api
command: [python, -m, messenger.relay]
restart: unless-stopped
healthcheck: {disable: true}
secrets: [database_url, cache_url, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
worker:
<<: *api
command: [python, -m, messenger.worker]
restart: unless-stopped
cpus: 2
mem_limit: 1536m
memswap_limit: 1536m
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
healthcheck: {disable: true}
secrets:
- database_url
- cache_url
- broker_url
- source: s3_worker_access_key_id
target: s3_access_key_id
- source: s3_worker_secret_access_key
target: s3_secret_access_key
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
s3:
image: mtusi-lab5-s3:2025-10-15
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
cap_drop: [ALL]
cpus: 1
mem_limit: 768m
memswap_limit: 768m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live']
interval: 2s
timeout: 1s
retries: 30
networks: [backend]
bucket-init:
image: mtusi-lab5-mc:2025-08-13
build: {context: ., dockerfile: infra/s3.Dockerfile, target: client}
command: [/bootstrap-s3.sh]
volumes:
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
- ./infra/api-policy.json:/policies/api-policy.json:ro
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
- ./infra/worker-policy.json:/policies/worker-policy.json:ro
- ./infra/test-policy.json:/policies/test-policy.json:ro
secrets: [s3_root_user, s3_root_password, s3_api_access_key_id, s3_api_secret_access_key,
s3_reconciler_access_key_id, s3_reconciler_secret_access_key,
s3_test_access_key_id, s3_test_secret_access_key,
s3_worker_access_key_id, s3_worker_secret_access_key]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
cpus: 0.5
mem_limit: 128m
networks: [backend]
depends_on:
s3: {condition: service_healthy}
reconciler:
<<: *api
profiles: [tools]
command: [python, -m, messenger.cleanup]
healthcheck: {disable: true}
secrets:
- database_url
- cache_url
- source: s3_reconciler_access_key_id
target: s3_access_key_id
- source: s3_reconciler_secret_access_key
target: s3_secret_access_key
depends_on:
migrate: {condition: service_completed_successfully}
bucket-init: {condition: service_completed_successfully}
networks:
backend: {}
volumes:
broker_data:
s3_data:
postgres_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_api_access_key_id:
file: ./secrets/s3_api_access_key_id
s3_api_secret_access_key:
file: ./secrets/s3_api_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
s3_test_access_key_id:
file: ./secrets/s3_test_access_key_id
s3_test_secret_access_key:
file: ./secrets/s3_test_secret_access_key
broker_url:
file: ./secrets/broker_url
s3_worker_access_key_id:
file: ./secrets/s3_worker_access_key_id
s3_worker_secret_access_key:
file: ./secrets/s3_worker_secret_access_key
+8
View File
@@ -0,0 +1,8 @@
{
"working_files_checked": 104,
"reachable_objects_checked": 128,
"requested_names_absent": true,
"actual_secret_values_absent": true,
"public_requirements_unchanged": true,
"upstream_push_disabled": true
}
+281
View File
@@ -0,0 +1,281 @@
{
"source_commit": "25e4ff977ee932e507524d565e10c2cb9de8296e",
"project": "mtusi-lab5-clean-1792259604",
"image": "mtusi-messenger:lab5-verified-1792259604",
"revision_label_matches": true,
"fresh_secrets_and_volumes": true,
"smoke_passed": true,
"own_cases_passed": 84,
"full_fault_matrix_passed": true,
"same_session_persistence_passed": true,
"all_services_restarted": [
"api",
"postgres",
"s3",
"broker",
"valkey",
"worker",
"relay",
"lb"
],
"before": {
"api": [
{
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
"started_at": "2026-10-17T17:54:22.360219637Z",
"pid": 86502,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
},
{
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
"started_at": "2026-10-17T17:56:41.112984132Z",
"pid": 1849,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
}
],
"postgres": [
{
"id": "2b1547e1e079a74d5b2c11917b8f865d45449daf67232cf93736c24d3c72d452",
"started_at": "2026-10-17T17:53:35.127862462Z",
"pid": 81760,
"ports": {
"5432/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "55437"
}
]
},
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db"
}
],
"s3": [
{
"id": "eed453c4decc6ffcf3e9691ad06f3d5beacbf32ac311a8d3df812f7fc2a23886",
"started_at": "2026-10-17T17:53:34.938232295Z",
"pid": 81607,
"ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59005"
}
]
},
"image": "sha256:96efc3b1fee5c9f362cd39d5c2287f903c67c66cfd8b1ee7a112242978b2822b"
}
],
"broker": [
{
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
"started_at": "2026-10-17T17:54:27.551506417Z",
"pid": 87340,
"ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
},
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
}
],
"valkey": [
{
"id": "d9bc56c3384fff2b3db1b898d8a09b9aaf077b238a01816c9cb7fc650de012ff",
"started_at": "2026-10-17T17:53:35.027799337Z",
"pid": 81685,
"ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56384"
}
]
},
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
}
],
"worker": [
{
"id": "fd613952c9ad7d7b71a779e38eb082eb78d449f2fecbac578ecaa2ab1786d74d",
"started_at": "2026-10-17T17:56:19.573687511Z",
"pid": 99682,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
}
],
"relay": [
{
"id": "edbf313e6e97397f63432afa4c9799ee9e510c6621b31a56dd6d6859c4b384f8",
"started_at": "2026-10-17T17:56:18.466104844Z",
"pid": 99383,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
}
],
"lb": [
{
"id": "3e1e4cef40a73f29ebd16dc44465cc1e3151b6c269b8f98f297ff5f3b516bea9",
"started_at": "2026-10-17T17:53:45.990409384Z",
"pid": 83113,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "28085"
}
]
},
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9"
}
]
},
"after": {
"api": [
{
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
"started_at": "2026-10-17T17:56:49.518161302Z",
"pid": 3995,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
},
{
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
"started_at": "2026-10-17T17:56:49.497895719Z",
"pid": 3914,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
}
],
"postgres": [
{
"id": "2b1547e1e079a74d5b2c11917b8f865d45449daf67232cf93736c24d3c72d452",
"started_at": "2026-10-17T17:56:49.373826427Z",
"pid": 3766,
"ports": {
"5432/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "55437"
}
]
},
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db"
}
],
"s3": [
{
"id": "eed453c4decc6ffcf3e9691ad06f3d5beacbf32ac311a8d3df812f7fc2a23886",
"started_at": "2026-10-17T17:56:49.225153469Z",
"pid": 3581,
"ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59005"
}
]
},
"image": "sha256:96efc3b1fee5c9f362cd39d5c2287f903c67c66cfd8b1ee7a112242978b2822b"
}
],
"broker": [
{
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
"started_at": "2026-10-17T17:56:49.426393761Z",
"pid": 3837,
"ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
},
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
}
],
"valkey": [
{
"id": "d9bc56c3384fff2b3db1b898d8a09b9aaf077b238a01816c9cb7fc650de012ff",
"started_at": "2026-10-17T17:56:49.328862136Z",
"pid": 3654,
"ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56384"
}
]
},
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773"
}
],
"worker": [
{
"id": "fd613952c9ad7d7b71a779e38eb082eb78d449f2fecbac578ecaa2ab1786d74d",
"started_at": "2026-10-17T17:56:49.806513386Z",
"pid": 4189,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
}
],
"relay": [
{
"id": "edbf313e6e97397f63432afa4c9799ee9e510c6621b31a56dd6d6859c4b384f8",
"started_at": "2026-10-17T17:56:49.443181386Z",
"pid": 3840,
"ports": {},
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1"
}
],
"lb": [
{
"id": "3e1e4cef40a73f29ebd16dc44465cc1e3151b6c269b8f98f297ff5f3b516bea9",
"started_at": "2026-10-17T17:56:49.161912927Z",
"pid": 3501,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "28085"
}
]
},
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9"
}
]
},
"module_sha256": {
"worker.py": "37168828d2d01ec13bd1a108785b1f5afeac007a3fa7790b6172599edfd2eb34",
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298",
"operator.py": "04b22eb35647e721b0460f251db6cd3a4f07826d141814e8142ca1b683a15755",
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
"cleanup.py": "4a5f08994bbe7defa997bd154baf2d973de68ca4aab934683a316b514a53430d",
"attachments.py": "8e6a0924809332b2020e3abc2c0b89d6dec6a83d287c264368eee3048a31f2e7",
"schemas.py": "5cca4d2ed64401fb2b9f530f3646f3bdfb979b82f050bd8d3126dfa6b5f62af7",
"relay.py": "7c1d565187d0f42ac08f17bfaf6bb4df770e42094845c438371513ce90bcc60d",
"pipeline.py": "37dfe1b31cdefdac10ffe6808b125d456623681edb886787898f0b8fe29501a7",
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
"images.py": "a01914d47ed69cee1605a3f2f9dffb9d90fa7850b8d2be5c37694af6ef8861cb",
"settings.py": "9ae56366d9156db93c888bf1451ac4c55d9cfa5f2a2b70a47dd9170ecf4af149",
"app.py": "2774f40d323a9d28839c47c7c0e7703d76b69d58af0ebf2222fb33b6df15435b",
"broker.py": "dfa9bc04bf84cb9f0277ce3a86b5ac8062ec96e69278070f84b581977ae20240",
"repository.py": "91ec54c8032ed283e39604ca0405a9752339096aa559498f2e9c7d099c8e47ba",
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f",
"faults.py": "f3bd0446a7024074e66df1aa07245f85824a94a24e9dafb4039a352e5bd42ec9"
},
"both_api_modules_match": true,
"api_ports_unpublished": true,
"private_values_absent_from_logs": true,
"checkout_clean_after_tests": true
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+26
View File
@@ -0,0 +1,26 @@
{
"retention_seconds": 300,
"failed_real_age_seconds": 301.00391,
"seed": {
"created_at": "2026-10-17T17:45:40.213510+00:00",
"active_id": "a406c829-1654-4e43-aab0-dd6f25df22ad",
"failed_id": "d93b3943-9788-4179-81f7-a7e9a4134ca1",
"orphan_id": "35f731d5-ab06-4c68-9a58-4b808ad4c1be"
},
"dry_run": {
"scanned": 118,
"eligible": 14,
"deleted": 0,
"protected": 49
},
"apply": {
"scanned": 118,
"eligible": 14,
"deleted": 14,
"protected": 49
},
"active_protected_then_ready": true,
"orphan_deleted": true,
"failed_source_expired": true,
"published_objects_unchanged": 4
}
+341
View File
@@ -0,0 +1,341 @@
{
"initial_instances": [
{
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
"started_at": "2026-10-17T17:48:24.316494638Z",
"pid": 61117,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "d06f1ce7a2ad48fb85a6e4c959495a6b"
},
{
"id": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
"started_at": "2026-10-17T17:45:53.909326888Z",
"pid": 45122,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "4d81bcd18cd841d58fb480df271130ec"
}
],
"failures": [
{
"duration_seconds": 5.945994583000356,
"workers": 6,
"requests": 2092,
"successes": 2031,
"errors": 61,
"successful_rps": 341.5744786930423,
"error_fraction": 0.029158699808795412,
"first_success_after_action_seconds": 1.9096203330009303,
"action_seconds": 1.902683000000252,
"p50_ms": 11.905290999493445,
"p95_ms": 57.65445899851329,
"p99_ms": 78.713124999922,
"instances": [
"4d81bcd18cd841d58fb480df271130ec",
"d06f1ce7a2ad48fb85a6e4c959495a6b"
],
"tail_instances": [
"4d81bcd18cd841d58fb480df271130ec"
],
"confirmed_writes": 201,
"stopped": {
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
"started_at": "2026-10-17T17:48:24.316494638Z",
"pid": 61117,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "d06f1ce7a2ad48fb85a6e4c959495a6b"
}
},
{
"duration_seconds": 5.482812500000364,
"workers": 6,
"requests": 1806,
"successes": 1691,
"errors": 115,
"successful_rps": 308.4183528070471,
"error_fraction": 0.0636766334440753,
"first_success_after_action_seconds": 1.8648894590005511,
"action_seconds": 1.440443999999843,
"p50_ms": 13.768250000794069,
"p95_ms": 50.30191599871614,
"p99_ms": 71.77345800118928,
"instances": [
"4d81bcd18cd841d58fb480df271130ec",
"99a8d6532148477f8630cabe3f0c88d8"
],
"tail_instances": [
"99a8d6532148477f8630cabe3f0c88d8"
],
"confirmed_writes": 165,
"stopped": {
"id": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
"started_at": "2026-10-17T17:45:53.909326888Z",
"pid": 45122,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "4d81bcd18cd841d58fb480df271130ec"
}
}
],
"concurrent_replays": 30,
"concurrent_effects": 1,
"rolling": {
"duration_seconds": 13.661339999998745,
"workers": 6,
"requests": 5950,
"successes": 5645,
"errors": 305,
"successful_rps": 413.2098315392574,
"error_fraction": 0.05126050420168067,
"first_success_after_action_seconds": 9.612477874999968,
"action_seconds": 9.61113162500078,
"p50_ms": 9.479749998718034,
"p95_ms": 53.65141699985543,
"p99_ms": 78.18379200034542,
"instances": [
"3e551e0479ef40d4ab399c2c97515ce0",
"94c58fccf2e045e196c07dd588a92c1a",
"98396ce0e5b4472c970589a37b35f629",
"99a8d6532148477f8630cabe3f0c88d8"
],
"tail_instances": [
"3e551e0479ef40d4ab399c2c97515ce0",
"94c58fccf2e045e196c07dd588a92c1a"
],
"confirmed_writes": 563,
"replacements": [
{
"old_container": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
"old_instance": "99a8d6532148477f8630cabe3f0c88d8",
"live_instances": [
"94c58fccf2e045e196c07dd588a92c1a",
"98396ce0e5b4472c970589a37b35f629"
]
},
{
"old_container": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
"old_instance": "98396ce0e5b4472c970589a37b35f629",
"live_instances": [
"3e551e0479ef40d4ab399c2c97515ce0",
"94c58fccf2e045e196c07dd588a92c1a"
]
}
],
"confirmed_messages_preserved": true
},
"persistence": {
"before": {
"postgres": {
"id": "45876dc9476e2b47667d1bde173e786cb20aba45a179caca86a374f732cba70b",
"started_at": "2026-10-17T17:38:39.558313298Z",
"pid": 7427,
"running": true,
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"5432/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "55437"
}
]
}
},
"valkey": {
"id": "4a11709684b5cdd4f6e4488ae0e11ef42085b4fdeace1e3c7cff88b05d0e9d1a",
"started_at": "2026-10-17T17:37:37.056307755Z",
"pid": 1093,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56384"
}
]
}
},
"s3": {
"id": "9ab4050036931ffa30695ee3e29e57a0d767660324585a39fc21e8a8456e8ffc",
"started_at": "2026-10-17T17:37:37.243124088Z",
"pid": 1260,
"running": true,
"image": "sha256:94aff3c743f63f1a00f1b8d0ea1553fd31f21ffe622a6e70eecae1026546ec3c",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59005"
}
]
}
},
"broker": {
"id": "e21054f56ca9160a2587b123cb21624e2685ebc7e252df354add56c704032f7b",
"started_at": "2026-10-17T17:46:01.597192669Z",
"pid": 46402,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
}
},
"lb": {
"id": "12688085914dbd56dce6dec01d5161bf371b45cec95a99b5510dc3782715f91a",
"started_at": "2026-10-17T17:38:48.530733677Z",
"pid": 8643,
"running": true,
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8085"
}
]
}
}
},
"after": {
"postgres": {
"id": "45876dc9476e2b47667d1bde173e786cb20aba45a179caca86a374f732cba70b",
"started_at": "2026-10-17T17:51:16.499957968Z",
"pid": 75615,
"running": true,
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"5432/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "55437"
}
]
}
},
"valkey": {
"id": "4a11709684b5cdd4f6e4488ae0e11ef42085b4fdeace1e3c7cff88b05d0e9d1a",
"started_at": "2026-10-17T17:51:16.385997843Z",
"pid": 75510,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56384"
}
]
}
},
"s3": {
"id": "9ab4050036931ffa30695ee3e29e57a0d767660324585a39fc21e8a8456e8ffc",
"started_at": "2026-10-17T17:51:16.228970093Z",
"pid": 75341,
"running": true,
"image": "sha256:94aff3c743f63f1a00f1b8d0ea1553fd31f21ffe622a6e70eecae1026546ec3c",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59005"
}
]
}
},
"broker": {
"id": "e21054f56ca9160a2587b123cb21624e2685ebc7e252df354add56c704032f7b",
"started_at": "2026-10-17T17:51:16.531699968Z",
"pid": 75682,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
}
},
"lb": {
"id": "12688085914dbd56dce6dec01d5161bf371b45cec95a99b5510dc3782715f91a",
"started_at": "2026-10-17T17:51:16.329691384Z",
"pid": 75442,
"running": true,
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8085"
}
]
}
}
},
"same_token_public_test": true
},
"final_instances": [
{
"id": "2dfd89da126d334f744a65b1db2e0a7da97fe437f5480bf8ad8730657deeb7d2",
"started_at": "2026-10-17T17:51:16.725000593Z",
"pid": 75960,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "aa801fbb4c3547bbb6d90db5fd31db59"
},
{
"id": "28997d75263b8c92dcd8afe813478c473ad1eef479ed996dce86916ad245c75e",
"started_at": "2026-10-17T17:51:16.571504468Z",
"pid": 75797,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "b22ea2f25f6546f88ddac407dd792c07"
}
]
}
+4
View File
@@ -0,0 +1,4 @@
.venv/bin/python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8085"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 2, "first_recovery_seconds": 0.984}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8085"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
File diff suppressed because it is too large Load Diff
+14
View File
@@ -0,0 +1,14 @@
{
"scope": "Редактура документации и метаданных рисунка",
"runtime_sources_unchanged": true,
"migrations_dependencies_and_infrastructure_unchanged": true,
"instructor_contract_and_public_tests_unchanged": true,
"measurement_values_unchanged": true,
"original_head": "dc11d065fb13d155c19612b95d9b42704da60f9f",
"rewritten_history_head": "f1dbe61eafc3c7254d2351a160ed63099cb38678",
"commit_map": {
"25e4ff977ee932e507524d565e10c2cb9de8296e": "1f432c2e848c89ca538c02aa5d7c41332d62b0c7",
"276dbdad8a34da4c5b57fbf1879ca169b581d75b": "276dbdad8a34da4c5b57fbf1879ca169b581d75b",
"dc11d065fb13d155c19612b95d9b42704da60f9f": "f1dbe61eafc3c7254d2351a160ed63099cb38678"
}
}
+2
View File
@@ -0,0 +1,2 @@
.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:8085"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+13
View File
@@ -0,0 +1,13 @@
============================= test session starts ==============================
platform darwin -- Python 3.14.7, pytest-9.1.1, pluggy-1.7.0
rootdir: <repository>
configfile: pyproject.toml
testpaths: student-tests
plugins: anyio-4.15.1
collected 84 items
student-tests/test_api.py .................................... [ 42%]
student-tests/test_cluster.py ............ [ 57%]
student-tests/test_pipeline.py .................................... [100%]
======================== 84 passed, 1 warning in 18.40s ========================
+33
View File
@@ -0,0 +1,33 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
set -eu
# Never print mc arguments/errors: they can include access key identifiers.
exec 3>&1
exec > /tmp/bootstrap.log 2>&1
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
bucket="$S3_BUCKET"
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
mc --config-dir /tmp/mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
for role in api worker reconciler test; do
target_bucket="$bucket"
policy_source=/policies/api-policy.json
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
if [ "$role" = worker ]; then policy_source=/policies/worker-policy.json; fi
if [ "$role" = test ]; then
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
policy_source=/policies/test-policy.json
target_bucket="$bucket-tests"
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
fi
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
key="$(cat /run/secrets/s3_${role}_access_key_id)"
mc --config-dir /tmp/mc admin user add local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
done
trap - EXIT
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
+44
View File
@@ -0,0 +1,44 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
timeout connect 500ms
timeout check 500ms
timeout client 20s
timeout server 30s
timeout http-request 3s
retries 0
resolvers docker
nameserver embedded 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 0s
hold nx 1s
hold refused 1s
hold timeout 1s
hold other 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
listen metrics
bind :8404
stats enable
stats uri /stats
stats refresh 1s
+26
View File
@@ -0,0 +1,26 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*",
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+18
View File
@@ -0,0 +1,18 @@
# Official security-fixed source release; no dependency on retired image tags.
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
COPY --from=builder /go/bin/minio /usr/local/bin/minio
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
COPY --from=builder /go/bin/mc /usr/local/bin/mc
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/bin/sh"]
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
exec minio "$@"
+27
View File
@@ -0,0 +1,27 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*",
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+34
View File
@@ -0,0 +1,34 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::BUCKET"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/staging/*",
"arn:aws:s3:::BUCKET/objects/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::BUCKET/objects/*"
]
}
]
}
+37
View File
@@ -0,0 +1,37 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id,user_id),
UNIQUE (chat_id,position)
);
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry ON sessions (expires_at);
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
chat_id uuid NOT NULL REFERENCES chats(id),
idempotency_key varchar(128) COLLATE "C" NOT NULL
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
response_message_id uuid NOT NULL REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
CHECK (expires_at > created_at)
);
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (id,chat_id),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
ALTER TABLE messages ADD UNIQUE (id,chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id,attachment_id),
UNIQUE (message_id,position),
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
);
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
+46
View File
@@ -0,0 +1,46 @@
-- Preserve earlier migration checksums; transition lab4 storage to versioned jobs.
ALTER TABLE attachments DROP CONSTRAINT attachments_storage_state_check;
ALTER TABLE attachments ADD CHECK (storage_state IN ('queued','processing','ready','failed')) NOT VALID;
UPDATE attachments SET storage_state='queued' WHERE storage_state IN ('pending','missing');
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(variants)='array');
ALTER TABLE attachments ADD COLUMN error_code text;
CREATE TABLE attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
source_key text NOT NULL,
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
fence uuid,
lease_until timestamptz,
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
error_code text,
source_expired boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (attachment_id,pipeline_version)
);
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until,next_attempt_at);
CREATE TABLE job_outbox (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
stream_id text
);
CREATE INDEX outbox_due ON job_outbox(due_at) WHERE published_at IS NULL;
CREATE TABLE pipeline_audit (
sequence bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
event text NOT NULL,
details jsonb NOT NULL DEFAULT '{}',
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Only an operator with DB access can configure faults; production ignores this table.
CREATE TABLE pipeline_faults (
job_id uuid REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
phase text NOT NULL,
action text NOT NULL CHECK (action IN ('crash','hang','transient','permanent')),
remaining integer NOT NULL DEFAULT 1 CHECK (remaining >= 0),
PRIMARY KEY(job_id,phase)
);
+8
View File
@@ -0,0 +1,8 @@
-- A paused API must not acknowledge an object that GC has already retired.
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY,
object_key text NOT NULL UNIQUE,
state text NOT NULL DEFAULT 'uploading' CHECK (state IN ('uploading','accepted','abandoned')),
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+46
View File
@@ -0,0 +1,46 @@
[project]
name = "mtusi-messenger"
version = "0.5.0"
description = "MTUSI lab 5: durable versioned image pipeline"
requires-python = ">=3.14,<3.15"
dependencies = [
"argon2-cffi==25.1.0",
"boto3==1.43.109",
"fastapi==0.143.3",
"pillow==13.0.0",
"psycopg[binary,pool]==3.3.6",
"pydantic==2.14.1",
"redis==8.1.0",
"uvicorn==0.54.0",
]
[build-system]
requires = ["hatchling==1.27.0"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/messenger"]
[dependency-groups]
dev = [
"httpx==0.28.1",
"openapi-spec-validator==0.7.2",
"pytest==9.1.1",
"ruff==0.17.1",
]
research = ["matplotlib==3.11.2"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra --tb=short"
[tool.ruff]
extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"]
target-version = "py314"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once, without printing their values."""
import os
import secrets
from pathlib import Path
root = Path(__file__).resolve().parents[1]
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_urlsafe(32) + "\n")
password = password_file.read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
"cache_url": "redis://valkey:6379/0",
"broker_url": "redis://broker:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n")
# Compose bind-mount secrets must be readable by the non-root container process.
# The parent directory remains 0700 on the host.
for role in ("root", "api", "worker", "reconciler", "test"):
names = (
("s3_root_user", "s3_root_password")
if role == "root"
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
)
for index, name in enumerate(names):
path = directory / name
if not path.exists():
path.write_text(
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
)
for name in (
"postgres_password",
"database_url",
"cache_url",
"broker_url",
*[p.name for p in directory.glob("s3_*")],
):
os.chmod(directory / name, 0o644)
env = root / ".env"
if not env.exists():
env.write_text((root / ".env.example").read_text())
print("Local secret files and environment are ready; existing values preserved.")
+82
View File
@@ -0,0 +1,82 @@
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
import json
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="http://127.0.0.1:8085", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+33
View File
@@ -0,0 +1,33 @@
"""Local test operator configuration. Credentials stay in ignored files/in memory."""
import os
from pathlib import Path
from psycopg.conninfo import make_conninfo
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
def settings(role="worker"):
prefix = "s3_root" if role == "root" else "s3_" + role
access, secret = (
("user", "password") if role == "root" else ("access_key_id", "secret_access_key")
)
return Settings(
make_conninfo(
host="127.0.0.1",
port=int(os.getenv("TEST_PG_PORT", "55437")),
user="messenger",
dbname="messenger",
password=(ROOT / "secrets/postgres_password").read_text().strip(),
),
"redis://127.0.0.1:" + os.getenv("TEST_CACHE_PORT", "56384") + "/0",
s3_endpoint="http://127.0.0.1:" + os.getenv("TEST_S3_PORT", "59005"),
s3_access_key=(ROOT / ("secrets/" + prefix + "_" + access)).read_text().strip(),
s3_secret_key=(ROOT / ("secrets/" + prefix + "_" + secret)).read_text().strip(),
broker_url="redis://127.0.0.1:" + os.getenv("TEST_BROKER_PORT", "56385") + "/0",
job_timeout=int(os.getenv("TEST_JOB_TIMEOUT_SECONDS", "5")),
fault_injection=True,
)
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# All running application/state services; no migration, down -v, flush or seed.
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 broker relay worker lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8084"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
import argparse
import hashlib
import io
import json
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose
from pipeline_lab import settings
from messenger.cleanup import collect
from messenger.pipeline import Pipeline, intent
from messenger.repository import PostgresRepository
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("action", choices=("seed", "verify"))
parser.add_argument("--seed", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
args = parser.parse_args()
config = settings("reconciler")
db = PostgresRepository(config.database_url, pool_size=3)
db.open()
store = ObjectStore(config)
try:
if args.action == "seed":
check(not args.seed.exists(), "Seed already exists")
compose("stop", "--timeout", "2", "worker", "relay")
try:
c = Client(args.base_url)
alice, _, _, room = c.bootstrap()
active, data = c.attachment(alice, room)
failed, _ = c.attachment(alice, room)
orphan = str(uuid4())
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
uploader = ObjectStore(settings("api"))
try:
uploader.put(
"staging/" + orphan,
io.BytesIO(data),
len(data),
"image/png",
hashlib.sha256(data).hexdigest(),
)
finally:
uploader.close()
with db.connection(write=True) as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
).fetchone()
conn.execute(
"UPDATE attachment_jobs SET "
"next_attempt_at=clock_timestamp()+interval '1 hour' "
"WHERE job_id=%s",
(j["job_id"],),
)
conn.execute(
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
"hour' WHERE job_id=%s",
(j["job_id"],),
)
f = conn.execute(
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
).fetchone()
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
"(%s,'before_effect','permanent')",
(f["job_id"],),
)
seed = {
"created_at": datetime.now(UTC).isoformat(),
"active_id": active["id"],
"failed_id": failed["id"],
"orphan_id": orphan,
}
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
print(
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
"credentials saved"
)
finally:
compose("start", "worker", "relay")
else:
check(args.output is not None and not args.output.exists(), "Choose a new output file")
seed = json.loads(args.seed.read_text())
# Real S3 timestamps and failed retention; no modified clocks.
while True:
with db.connection() as conn:
f = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
).fetchone()
check(f["state"] == "failed", "Failed seed did not reach terminal state")
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
if age >= 301:
break
check(age >= 0, "Clock mismatch")
time.sleep(min(1, 301 - age))
with db.connection() as conn:
ready = conn.execute(
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
).fetchone()
check(ready is not None, "Need a published attachment to protect")
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
before = {key: store.head(key)["ETag"] for key in keys}
preview = collect(db, store, 300, False)
result = collect(db, store, 300, True)
check(
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
)
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
check(
store.head("staging/" + seed["failed_id"]) is None,
"Expired failed source not deleted",
)
check(
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
"Published result changed",
)
with db.connection(write=True) as conn:
expired = conn.execute(
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
(seed["failed_id"],),
).fetchone()
check(expired["source_expired"], "Expired source not registered")
j = conn.execute(
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
"attachment_id=%s RETURNING job_id",
(seed["active_id"],),
).fetchone()
intent(conn, j["job_id"])
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if db.attachment(seed["active_id"])["storage_state"] == "ready":
break
time.sleep(0.2)
check(
db.attachment(seed["active_id"])["storage_state"] == "ready",
"Protected work failed to complete",
)
proof = {
"retention_seconds": 300,
"failed_real_age_seconds": age,
"seed": seed,
"dry_run": preview,
"apply": result,
"active_protected_then_ready": True,
"orphan_deleted": True,
"failed_source_expired": True,
"published_objects_unchanged": len(keys),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(proof, indent=2) + "\n")
print("PASS: real-age cleanup preserves active work and published results")
finally:
store.close()
db.close()
if __name__ == "__main__":
main()
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "s3", "broker", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()
+513
View File
@@ -0,0 +1,513 @@
#!/usr/bin/env python3
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
import argparse
import hashlib
import io
import json
import subprocess
import sys
import time
from pathlib import Path
from uuid import uuid4
from botocore.exceptions import ClientError
from cluster import ROOT, compose, inspect, instances, wait_lb
from PIL import Image
from pipeline_lab import settings
from messenger.broker import STREAM, Broker, Job
from messenger.pipeline import Pipeline
from messenger.repository import PostgresRepository, stamp
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def until(fn, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = fn()
if result:
return result
time.sleep(0.2)
raise AssertionError("Timed out waiting for a durable pipeline condition")
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
args = parser.parse_args()
if args.output.exists():
parser.error("Output already exists; choose a new evidence file")
config = settings()
db = PostgresRepository(config.database_url, pool_size=4)
db.open()
broker, store = Broker(config), ObjectStore(config)
c = Client(args.base_url)
alice, bob, eve, room = c.bootstrap()
records, accepted = [], []
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
def query(sql, params=()):
with db.connection() as conn:
return conn.execute(sql, params).fetchall()
def job(a, version=1):
return query(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
(a["id"], version),
)[0]
def audit(j):
return query(
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
(j["job_id"],),
)
def injected(j, phase):
return any(
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
)
def fault(j, phase, action, count=1):
with db.connection(write=True) as conn:
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
(j["job_id"], phase, action, count),
)
def upload():
a, original = c.attachment(alice, room)
check(original == data, "Unexpected fixture")
accepted.append(a["id"])
return a
def ready(a):
value = c.wait_ready(bob, a)
path = "/api/v1/attachments/" + a["id"]
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
for v in value["variants"]:
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
with Image.open(io.BytesIO(content)) as im:
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
check(len(value["variants"]) == 3, "Missing variants")
check(
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
"Unexpected duplicate link",
)
return value
def saved(name, a, start, **details):
j = job(a, details.get("version", 1))
current = query(
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
)[0]
keys = [
o["Key"]
for o in store.client.list_objects_v2(
Bucket=store.bucket,
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
).get("Contents", [])
]
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
record = {
"scenario": name,
"attachment_id": a["id"],
"job_id": str(j["job_id"]),
"elapsed_seconds": time.monotonic() - start,
"final_state": current["storage_state"],
"attempts": j["attempt"],
"final_version": current["pipeline_version"],
"published_objects": len(keys),
"audit": audit(j),
**details,
}
records.append(record)
print("PASS: " + name, flush=True)
def envelope(j, attempt=None):
return Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=str(j["attachment_id"]),
pipeline_version=j["pipeline_version"],
attempt=attempt or max(1, j["attempt"]),
created_at=stamp(j["created_at"]),
)
try:
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
check(
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
is None,
"Gap was not established",
)
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
before = instances()
compose("restart", "--timeout", "3", "api")
wait_lb(args.base_url)
after = instances()
check(
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
"API did not restart",
)
compose("start", "relay")
until(
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
)
entry_ids = [
id
for id, fields in broker.client.xrange(STREAM)
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
]
check(bool(entry_ids), "Broker has no accepted work")
broker_id = compose("ps", "-q", "broker").strip()
broker_before = inspect(broker_id)
subprocess.run(
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
)
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
def broker_restored():
try:
return broker.client.ping()
except Exception:
return False
until(broker_restored)
check(
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
"Durable stream lost confirmed work",
)
compose("start", "worker")
ready(a)
history = c.call(
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
)["json"]
check(message in history["items"], "Queued attachment message lost")
saved(
"commit_publish_gap_api_restart_broker_sigkill",
a,
start,
api_before=before,
api_after=after,
broker_before=broker_before,
broker_after=inspect(broker_id),
broker_entry_ids=entry_ids,
message_id=message["id"],
)
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "after_publish", "crash")
compose("start", "relay")
until(lambda: injected(j, "after_publish"))
until(
lambda: (
sum(
json.loads(f["job"])["job_id"] == str(j["job_id"])
for _, f in broker.client.xrange(STREAM)
)
>= 2
)
)
compose("start", "worker")
ready(a)
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
saved("publish_before_outbox_commit_duplicate", a, start)
for phase in ("before_effect", "after_effect", "after_commit"):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, phase, "crash")
compose("start", "worker")
ready(a)
until(lambda j=j, phase=phase: injected(j, phase))
if phase == "after_commit":
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
check(job(a)["attempt"] == 1, "Committed job was processed twice")
else:
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
saved("worker_crash_" + phase, a, start)
for action, count, expected in (
("transient", 2, "ready"),
("hang", 1, "ready"),
("permanent", 1, "failed"),
("transient", 5, "failed"),
):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "before_effect", action, count)
compose("start", "worker")
path = "/api/v1/attachments/" + a["id"]
if expected == "failed":
until(
lambda path=path: (
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
== "failed"
)
)
terminal = job(a)
check(terminal["attempt"] == count, "Wrong retry budget")
c.call("GET", path + "/content", 409, actor=bob)
c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/messages",
409,
actor=alice,
body={"text": "", "attachment_ids": [a["id"]]},
headers={"Idempotency-Key": uuid4().hex},
)
# Explicit CLI replay, same logical job; no public administrator endpoint.
replay = json.loads(
compose(
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
)
)
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
else:
terminal = None
ready(a)
events = audit(j)
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
rejected = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
],
cwd=ROOT,
capture_output=True,
)
check(rejected.returncode != 0, "Ready replay was accepted")
saved(
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
a,
start,
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
if terminal
else None,
backoff_delays=delays,
)
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
old = job(a)
old_envelope = envelope(old, 1)
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
check(outcome == "claimed", "Cannot establish old writer")
Pipeline(db, config).replay(a["id"], new_version=True)
new = job(a, 2)
# Publish v2 first, then v1, plus duplicates through both worker slots.
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
broker.publish(value)
compose("start", "worker", "relay")
current = ready(a)
row = db.attachment(a["id"])
check(
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
"Old writer overwrote v2",
)
for _ in range(20):
broker.publish(old_envelope)
broker.publish(envelope(new, 1))
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
check(
db.attachment(a["id"])["object_key"] == row["object_key"],
"Stale delivery changed current pointer",
)
check(
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
== current,
"Old version mutated ready metadata",
)
saved(
"reorder_stale_v1_after_v2_and_publication_fence",
a,
start,
version=2,
old_audit=audit(old),
duplicates=44,
)
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
a = {"id": accepted[0]}
for suffix in ("", "/content", "/content?variant=crop"):
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
# Direct bucket access and write/delete privilege checks use no public credentials.
import boto3
from botocore import UNSIGNED
from botocore.config import Config
first = db.attachment(accepted[0])
unsigned = boto3.client(
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
)
try:
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
raise AssertionError("Anonymous S3 read succeeded")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
)
finally:
unsigned.close()
denied = []
for role, action in (
("api", "put_object"),
("api", "delete_object"),
("worker", "delete_object"),
("reconciler", "put_object"),
):
scoped = ObjectStore(settings(role))
try:
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
if action == "put_object":
kw["Body"] = b"forbidden"
try:
getattr(scoped.client, action)(**kw)
raise AssertionError("IAM action unexpectedly allowed")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
"Wrong IAM status",
)
denied.append(role + ":" + action)
finally:
scoped.close()
# Stop the actual uploader, then read original and thumbnail through another replica.
response = c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
202,
actor=alice,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
cross = response["json"]
accepted.append(cross["id"])
ready(cross)
uploader = response["headers"]["X-Instance-Id"]
process = next(v for v in instances() if v["instance"] == uploader)
subprocess.run(
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
)
try:
wait_lb(args.base_url, count=1)
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
check(
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
"Other replica could not serve the original",
)
c.call(
"GET",
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
actor=bob,
)
cross_proof = {
"uploader": uploader,
"stopped": inspect(process["id"]),
"reader": download["headers"]["X-Instance-Id"],
}
finally:
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
wait_lb(args.base_url)
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
check(
not any(
actor["token"] in logs or actor["registration"]["password"] in logs
for actor in (alice, bob, eve)
),
"Credential in logs",
)
states = query(
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
)
check(
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
"Confirmed work lost",
)
stats = compose(
"exec",
"-T",
"worker",
"python",
"-c",
"import json;from pathlib import Path;print(json.dumps({"
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
)
result = {
"anonymous_s3_status": 403,
"denied_iam_actions": denied,
"cross_replica": cross_proof,
"track": 2,
"delivery": "at-least-once",
"effect_key": ["attachment_id", "pipeline_version", "variant"],
"worker_concurrency": config.worker_concurrency,
"prefetch_per_slot": 1,
"job_timeout_seconds": config.job_timeout,
"lease_seconds": config.lease_seconds,
"max_attempts": config.job_max_attempts,
"scenarios": records,
"accepted": len(accepted),
"lost_confirmed": 0,
"stale_overwrites": 0,
"worker_cgroup": json.loads(stats),
"logs_credentials_absent": True,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
finally:
compose("start", "worker", "relay", "broker", "api")
broker.close()
store.close()
db.close()
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""One-process implementation of the lab 1 messenger contract."""
+24
View File
@@ -0,0 +1,24 @@
import logging
import os
import uvicorn
def main():
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
logging.basicConfig(level=level, format="%(message)s")
uvicorn.run(
"messenger.app:create_app",
factory=True,
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
access_log=False,
log_level=level.lower(),
timeout_keep_alive=2,
timeout_graceful_shutdown=5,
)
if __name__ == "__main__":
main()
+348
View File
@@ -0,0 +1,348 @@
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
import json
import logging
import os
import re
from contextlib import asynccontextmanager
from time import perf_counter
from typing import Annotated
from urllib.parse import quote
from uuid import uuid4
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.exceptions import HTTPException
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from messenger.attachments import Attachments
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
from messenger.cache import SessionCache
from messenger.domain import DomainError, User, invalid, not_found
from messenger.repository import PostgresRepository
from messenger.schemas import (
AttachmentView,
ChatPage,
ChatView,
CreateChat,
CreateMessage,
CreateUser,
MessagePage,
MessageView,
SessionView,
UserView,
canonical_uuid,
)
from messenger.service import Messenger
from messenger.settings import Settings
from messenger.storage import ObjectStore
logger = logging.getLogger("messenger")
class RequestContext:
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id
async def __call__(self, scope: Scope, receive: Receive, send: Send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
request_id = str(uuid4())
scope.setdefault("state", {})["request_id"] = request_id
start, status, response_started = perf_counter(), 500, False
async def with_id(message: Message):
nonlocal status, response_started
if message["type"] == "http.response.start":
response_started = True
status = message["status"]
message["headers"] = [
pair
for pair in message.get("headers", [])
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
] + [
(b"x-request-id", request_id.encode()),
(b"x-instance-id", self.instance_id.encode()),
]
await send(message)
try:
await self.app(scope, receive, with_id)
except Exception as exc:
# Do not let Uvicorn print exception values that could contain user data.
if response_started:
raise
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
response = error_response(
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
)
await response(scope, receive, with_id)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request_id,
"method": scope["method"],
"route": route,
"status": status,
"duration_ms": round((perf_counter() - start) * 1000, 3),
}
)
)
def error_response(
request: Request, status: int, code: str, message: str, headers: dict | None = None
) -> JSONResponse:
identity = getattr(request.state, "request_id", str(uuid4()))
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": identity}},
headers={"X-Request-Id": identity, **(headers or {})},
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings.from_env()
repository = PostgresRepository(
settings.database_url,
settings.pool_size,
settings.idempotency_ttl,
settings.read_autocommit,
)
cache = SessionCache(settings.cache_url)
store = ObjectStore(settings)
authentication = AuthService(repository, cache, settings.session_ttl)
@asynccontextmanager
async def lifespan(app: FastAPI):
repository.open()
app.state.ready = True
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
try:
yield
finally:
app.state.ready = False
repository.close()
cache.close()
store.close()
logger.info(json.dumps({"event": "stopped"}))
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
app.state.messenger = Messenger(repository)
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
app.state.repository = repository
app.state.authentication = authentication
app.state.cache = cache
app.state.ready = False
@app.exception_handler(DomainError)
async def domain_error(request: Request, exc: DomainError):
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
@app.exception_handler(HTTPException)
async def http_error(request: Request, exc: HTTPException):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
@app.exception_handler(Exception)
async def unexpected_error(request: Request, exc: Exception):
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
def service(request: Request) -> Messenger:
return request.app.state.messenger
def token_digest(request: Request) -> bytes:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized()
return bearer_digest(request.headers.get("authorization"))
def identity(
digest: Annotated[bytes, Depends(token_digest)],
) -> Identity:
return authentication.identify(digest)
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
return current.user
async def credentials(request: Request) -> tuple[str, str]:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized("Basic")
result = basic_credentials(request.headers.get("authorization"))
if await request.body():
raise invalid("Вход не принимает тело запроса")
return result
def resource_id(chat_id: str) -> str:
try:
return canonical_uuid(chat_id)
except ValueError:
raise invalid("Некорректный UUID чата") from None
def pagination(request: Request) -> tuple[int, str | None]:
for key in ("limit", "cursor"):
if len(request.query_params.getlist(key)) > 1:
raise invalid("Параметр пагинации указан несколько раз")
raw = request.query_params.get("limit", "50")
try:
if not re.fullmatch(r"[0-9]+", raw):
raise ValueError
limit = int(raw)
if not 1 <= limit <= 100:
raise ValueError
except ValueError:
raise invalid("limit должен быть целым от 1 до 100") from None
return limit, request.query_params.get("cursor")
def json_input(request: Request):
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media != "application/json":
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
def message_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
raise invalid("Требуется корректный Idempotency-Key")
return values[0]
CurrentUser = Annotated[User, Depends(actor)]
Service = Annotated[Messenger, Depends(service)]
ChatId = Annotated[str, Depends(resource_id)]
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready(request: Request):
if not request.app.state.ready:
raise DomainError(503, "unavailable", "Приложение не готово")
repository.ready()
store.ready()
return {"status": "ok"}
@app.post(
"/api/v1/users",
status_code=201,
response_model=UserView,
dependencies=[Depends(json_input)],
)
def register(body: CreateUser):
return authentication.register(body.username, body.display_name, body.password)
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
return authentication.login(*value)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(current: Annotated[Identity, Depends(identity)]):
authentication.logout(current)
return Response(status_code=204)
@app.get("/api/v1/users/me", response_model=UserView)
def me(current: CurrentUser):
return current
@app.post(
"/api/v1/chats",
status_code=201,
response_model=ChatView,
dependencies=[Depends(json_input)],
)
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
return messenger.create_chat(current, body.title, body.member_ids)
@app.get("/api/v1/chats", response_model=ChatPage)
def chats(current: CurrentUser, messenger: Service, page: Pagination):
return messenger.chats(current, *page)
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
return messenger.chat(current, chat)
@app.post(
"/api/v1/chats/{chat_id}/messages",
status_code=201,
response_model=MessageView,
dependencies=[Depends(json_input)],
)
def send_message(
body: CreateMessage,
current: CurrentUser,
chat: ChatId,
messenger: Service,
key: Annotated[str, Depends(message_key)],
):
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
return messenger.messages(current, chat, *page)
def attachment_id(attachment_id: str) -> str:
try:
return canonical_uuid(attachment_id)
except ValueError:
raise invalid("Некорректный UUID вложения") from None
AttachmentId = Annotated[str, Depends(attachment_id)]
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentView)
async def upload(request: Request, current: CurrentUser, chat: ChatId):
return await app.state.attachments.upload(request, current, chat)
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
def attachment(current: CurrentUser, id: AttachmentId):
return app.state.attachments.metadata(current, id)
@app.get("/api/v1/attachments/{attachment_id}/content")
def content(request: Request, current: CurrentUser, id: AttachmentId):
row = app.state.attachments.record(current, id)
if row["storage_state"] != "ready":
raise DomainError(409, "conflict", "Вложение пока недоступно")
values = request.query_params.getlist("variant")
if (
len(values) > 1
or values
and values[0] not in ("original", "thumbnail", "crop", "optimized")
):
raise invalid("Некорректный вариант вложения")
if values and values[0] != "original":
row = next((v for v in row["variants"] if v["name"] == values[0]), None)
if row is None:
raise not_found()
row = {**row, "filename": values[0]}
body = store.download(row)
def chunks():
try:
while data := body.read(65536):
yield data
finally:
body.close()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "attachment; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
},
)
return app
+127
View File
@@ -0,0 +1,127 @@
"""Bounded upload: verified staging and transactional job intent precede HTTP 202."""
import asyncio
import hashlib
import tempfile
import unicodedata
import warnings
from anyio import to_thread
from PIL import Image, UnidentifiedImageError
from starlette.requests import ClientDisconnect
from messenger.domain import DomainError, invalid, new_id, not_found
from messenger.pipeline import Pipeline
from messenger.repository import attachment_value
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
def filename(value):
if (
not value
or not value.strip()
or len(value) > 255
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
):
raise invalid("Некорректное имя файла")
return value
def image_type(file, maximum):
try:
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > maximum:
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
if image.format not in ("PNG", "JPEG"):
raise ValueError
media = "image/png" if image.format == "PNG" else "image/jpeg"
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load() # Reject truncated pixel streams, not just an intact header.
file.seek(0)
return media
except Image.DecompressionBombError, Image.DecompressionBombWarning:
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
class Attachments:
def __init__(self, repository, messenger, store, settings):
self.repository, self.messenger, self.store = repository, messenger, store
self.settings = settings
self.slots = asyncio.Semaphore(settings.upload_slots)
async def upload(self, request, actor, chat_id):
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
values = request.query_params.getlist("filename")
if len(values) != 1:
raise invalid("Требуется одно имя файла")
name = filename(values[0])
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media not in ("image/png", "image/jpeg"):
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
length = request.headers.get("content-length")
if length is not None:
if not length.isascii() or not length.isdecimal():
raise invalid("Некорректный Content-Length")
if int(length) > self.settings.max_upload_bytes:
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
try:
await asyncio.wait_for(self.slots.acquire(), timeout=5)
except TimeoutError:
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
try:
with tempfile.TemporaryFile() as file:
size, digest = 0, hashlib.sha256()
try:
async with asyncio.timeout(15):
async for chunk in request.stream():
size += len(chunk)
if size > self.settings.max_upload_bytes:
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
digest.update(chunk)
await to_thread.run_sync(file.write, chunk)
except TimeoutError:
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
except ClientDisconnect:
raise invalid("Загрузка прервана") from None
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
if actual != media:
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
id = new_id()
key = "staging/" + id
await to_thread.run_sync(Pipeline(self.repository, self.settings).reserve, id, key)
await to_thread.run_sync(
self.store.put, key, file, size, actual, digest.hexdigest()
)
# A failure before this COMMIT leaves only an unaccepted staging orphan.
# The same transaction creates attachment, job and outbox intent.
row = await to_thread.run_sync(
Pipeline(self.repository, self.settings).accept,
id,
chat_id,
actor.id,
name,
actual,
size,
digest.hexdigest(),
key,
)
return attachment_value(row)
finally:
self.slots.release()
def record(self, actor, id):
row = self.repository.attachment(id)
if row is None:
raise not_found()
self.messenger.chat(actor, str(row["chat_id"]))
return row
def metadata(self, actor, id):
return attachment_value(self.record(actor, id))
+127
View File
@@ -0,0 +1,127 @@
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
import base64
import binascii
import hashlib
import re
import secrets
from contextlib import contextmanager
from dataclasses import dataclass
from threading import BoundedSemaphore
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.profiles import RFC_9106_LOW_MEMORY
from messenger.cache import SessionCache
from messenger.domain import DomainError, User
from messenger.repository import PostgresRepository
def unauthorized(scheme: str = "Bearer") -> DomainError:
challenge = scheme + ' realm="messenger"'
if scheme == "Basic":
challenge += ', charset="UTF-8"'
return DomainError(
401,
"unauthorized",
"Требуется действительная идентификация",
{"WWW-Authenticate": challenge},
)
def basic_credentials(header: str | None) -> tuple[str, str]:
try:
if header is None or len(header) > 2048:
raise ValueError
scheme, encoded = header.split(" ", 1)
if scheme.lower() != "basic":
raise ValueError
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = raw.split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
raise ValueError
if not 12 <= len(password) <= 128 or not password.strip():
raise ValueError
return username, password
except ValueError, UnicodeError, binascii.Error:
raise unauthorized("Basic") from None
def bearer_digest(header: str | None) -> bytes:
if header:
parts = header.split(" ")
if len(parts) == 2 and parts[0].lower() == "bearer":
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
return hashlib.sha256(parts[1].encode()).digest()
raise unauthorized()
@dataclass(frozen=True)
class Identity:
user: User
token_hash: bytes
@dataclass(frozen=True)
class Session:
token: str
token_type: str
expires_at: str
user: User
class AuthService:
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
self.repository, self.cache, self.ttl = repository, cache, ttl
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
self.kdf_slots = BoundedSemaphore(2)
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
@contextmanager
def kdf(self):
if not self.kdf_slots.acquire(timeout=0.5):
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
try:
yield
finally:
self.kdf_slots.release()
def register(self, username: str, display_name: str, password: str) -> User:
with self.kdf():
encoded = self.hasher.hash(password)
return self.repository.add_user(username, display_name, encoded)
def login(self, username: str, password: str) -> Session:
credentials = self.repository.credentials(username)
encoded = credentials[1] if credentials else self.dummy_hash
try:
with self.kdf():
self.hasher.verify(encoded, password)
except VerificationError, InvalidHashError:
raise unauthorized("Basic") from None
if credentials is None:
raise unauthorized("Basic")
token = secrets.token_urlsafe(32)
digest = hashlib.sha256(token.encode()).digest()
user = credentials[0]
expires = self.repository.create_session(digest, user.id, self.ttl)
return Session(token, "Bearer", expires, user)
def identify(self, digest: bytes) -> Identity:
session = self.repository.active_session(digest)
if session is None:
raise unauthorized()
user_id = str(session["user_id"])
user = self.cache.get(digest, user_id)
if user is None:
user = self.repository.get_user(user_id)
if user is None:
raise unauthorized()
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
self.cache.put(digest, user, ttl)
return Identity(user, digest)
def logout(self, identity: Identity):
self.repository.revoke_session(identity.token_hash)
self.cache.delete(identity.token_hash)
+96
View File
@@ -0,0 +1,96 @@
"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
import re
from datetime import datetime
from typing import Literal
from uuid import UUID
import redis
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
STREAM = "attachment-jobs"
GROUP = "image-workers-v1"
DLQ = "invalid-jobs"
class Job(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal[1]
job_id: str
type: Literal["attachment.process.v1"]
attachment_id: str
pipeline_version: int = Field(ge=1)
attempt: int = Field(ge=1)
created_at: str
traceparent: str | None = Field(
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
)
tracestate: str | None = Field(default=None, max_length=512)
@model_validator(mode="before")
@classmethod
def strict_carrier(cls, value):
if isinstance(value, dict):
if type(value.get("schema_version")) is not int:
raise ValueError("Integer schema version required")
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
raise ValueError("Optional carrier fields must be strings when provided")
return value
@field_validator("job_id", "attachment_id")
@classmethod
def uuid(cls, value):
if str(UUID(value)) != value:
raise ValueError("Canonical UUID required")
return value
@field_validator("created_at")
@classmethod
def timestamp(cls, value):
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
raise ValueError("RFC3339 required")
datetime.fromisoformat(value.replace("Z", "+00:00"))
return value
class Broker:
def __init__(self, settings):
self.client = redis.Redis.from_url(
settings.broker_url,
decode_responses=True,
socket_timeout=2,
socket_connect_timeout=1,
retry_on_timeout=False,
)
def group(self):
try:
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
def publish(self, job):
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
def ack(self, id):
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
self.client.eval(
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
1,
STREAM,
GROUP,
id,
)
def quarantine(self, id, reason):
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
with self.client.pipeline(transaction=True) as tx:
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
tx.xack(STREAM, GROUP, id)
tx.xdel(STREAM, id)
tx.execute()
def close(self):
self.client.close()
+57
View File
@@ -0,0 +1,57 @@
"""Disposable session-user cache; PostgreSQL always decides token validity."""
import json
import logging
from dataclasses import asdict
from redis import Redis
from redis.backoff import NoBackoff
from redis.exceptions import RedisError
from redis.retry import Retry
from messenger.domain import User
logger = logging.getLogger("messenger")
class SessionCache:
def __init__(self, url: str):
self.client = Redis.from_url(
url,
socket_connect_timeout=0.2,
socket_timeout=0.2,
max_connections=8,
decode_responses=True,
retry_on_timeout=False,
retry=Retry(NoBackoff(), 0),
)
@staticmethod
def key(digest: bytes) -> str:
return "session-user:" + digest.hex()
def get(self, digest: bytes, user_id: str) -> User | None:
try:
raw = self.client.get(self.key(digest))
if raw:
value = User(**json.loads(raw))
if value.id == user_id:
return value
except RedisError, ValueError, TypeError:
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
return None
def put(self, digest: bytes, user: User, ttl: int):
try:
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
def delete(self, digest: bytes):
try:
self.client.delete(self.key(digest))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
def close(self):
self.client.close()
+122
View File
@@ -0,0 +1,122 @@
"""Dry-run by default. Retention GC protects all active jobs and every published object."""
import argparse
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.pipeline import event
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def collect(db, store, retention, apply=False):
if retention < 300:
raise ValueError("Retention must be at least 300 seconds")
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
with db.connection() as lock:
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
return {**counts, "busy": True}
try:
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
for obj in page.get("Contents", []):
counts["scanned"] += 1
key = obj["Key"]
parts = key.split("/")
try:
if (
parts[0] not in ("staging", "objects")
or str(UUID(parts[1])) != parts[1]
):
continue
except ValueError, IndexError:
continue
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
continue
with db.connection(write=True) as conn:
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
).fetchone()
if (
reservation
and reservation["state"] == "uploading"
and reservation["expires_at"] > datetime.now(UTC)
):
counts["protected"] += 1
continue
# This row lock serializes replay/new-version with deletion eligibility.
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
).fetchone()
active = conn.execute(
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
"state IN ('queued','processing') LIMIT 1",
(parts[1],),
).fetchone()
published = (
a
and a["storage_state"] == "ready"
and (
key == a["object_key"]
or any(v["object_key"] == key for v in a["variants"])
)
)
recent_failed = conn.execute(
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
"state='failed' "
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
(parts[1], retention),
).fetchone()
if active or published or recent_failed:
counts["protected"] += 1
continue
head = store.head(key)
if (
not head
or head["ETag"] != obj["ETag"]
or head["ContentLength"] != obj["Size"]
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
< retention
):
continue
counts["eligible"] += 1
if apply:
if reservation and reservation["state"] == "uploading":
conn.execute(
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
(parts[1],),
)
store.client.delete_object(Bucket=store.bucket, Key=key)
counts["deleted"] += 1
rows = conn.execute(
"UPDATE attachment_jobs SET source_expired=true WHERE "
"source_key=%s "
"AND state='failed' RETURNING job_id",
(key,),
).fetchall()
for row in rows:
event(conn, row["job_id"], "source_expired")
finally:
lock.execute("SELECT pg_advisory_unlock(2301005)")
return counts
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
store = ObjectStore(settings)
try:
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
finally:
store.close()
db.close()
if __name__ == "__main__":
main()
+41
View File
@@ -0,0 +1,41 @@
"""Versioned keyset position, bound to a list and its resource, never an offset."""
import base64
import binascii
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.domain import HistoryKey, invalid
def encode_cursor(scope: str, position: HistoryKey) -> str:
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
return (
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
.decode()
.rstrip("=")
)
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
if value is None:
return None
try:
if not value or len(value) > 1024:
raise ValueError
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
data = json.loads(raw)
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
raise ValueError
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
raise ValueError
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
raise ValueError
stamp = datetime.fromisoformat(data["created_at"])
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
raise ValueError
identity = str(UUID(data["id"]))
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
raise invalid("Некорректный cursor или другая область пагинации") from None
+79
View File
@@ -0,0 +1,79 @@
"""Immutable domain values; no dependency on HTTP or a storage technology."""
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import uuid4
def utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
def new_id() -> str:
return str(uuid4())
class DomainError(Exception):
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid(message: str = "Некорректный запрос") -> DomainError:
return DomainError(400, "invalid_request", message)
def not_found() -> DomainError:
return DomainError(404, "not_found", "Объект не найден")
@dataclass(frozen=True, slots=True)
class User:
id: str
username: str
display_name: str
created_at: str
@dataclass(frozen=True, slots=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: str
@dataclass(frozen=True, slots=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: str
attachment_ids: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class Attachment:
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
created_at: str
updated_at: str
status: str = "ready"
variants: tuple = ()
error_code: str | None = None
type HistoryKey = tuple[str, str]
def history_key(item: Chat | Message) -> HistoryKey:
return item.created_at, item.id
+35
View File
@@ -0,0 +1,35 @@
"""Operator-only, single-use fault barriers. Disabled in the normal Compose stack."""
import os
import time
from messenger.domain import DomainError
from messenger.pipeline import event
class PermanentFailure(Exception):
pass
def checkpoint(db, settings, job, phase):
if not settings.fault_injection:
return
with db.connection(write=True) as conn:
row = conn.execute(
"UPDATE pipeline_faults SET remaining=remaining-1 WHERE job_id=%s AND phase=%s "
"AND remaining>0 RETURNING action",
(job, phase),
).fetchone()
if row:
event(conn, job, "fault_injected", phase=phase, action=row["action"], pid=os.getpid())
if not row:
return
match row["action"]:
case "crash":
os._exit(87)
case "hang":
time.sleep(settings.job_timeout * 3)
case "transient":
raise DomainError(503, "unavailable", "Synthetic transient failure")
case "permanent":
raise PermanentFailure
+51
View File
@@ -0,0 +1,51 @@
"""Pure, versioned image transforms. No network, filenames, or EXIF in output."""
import io
from PIL import Image, ImageOps
def fit(image, maximum):
scale = max(image.size)
if scale <= maximum:
return image.copy()
size = tuple(max(1, n * maximum // scale) for n in image.size)
return image.resize(size, Image.Resampling.LANCZOS)
def variants(source, version):
if version not in (1, 2):
raise ValueError("Unsupported pipeline version")
with Image.open(source) as raw:
oriented = ImageOps.exif_transpose(raw)
rgba = oriented.convert("RGBA")
image = Image.new("RGB", rgba.size, "white")
image.paste(rgba, mask=rgba.getchannel("A"))
rgba.close()
oriented.close()
try:
for name, output, format, options in (
("thumbnail", fit(image, 256), "JPEG", {"quality": 85, "subsampling": 0}),
(
"crop",
ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS),
"JPEG",
{"quality": 85, "subsampling": 0},
),
(
"optimized",
fit(image, 1280),
"WEBP",
{"quality": 82 if version == 1 else 76, "method": 4},
),
):
with output, io.BytesIO() as buffer:
output.save(buffer, format=format, exif=b"", **options)
yield (
name,
buffer.getvalue(),
output.size,
"image/webp" if format == "WEBP" else "image/jpeg",
)
finally:
image.close()
+46
View File
@@ -0,0 +1,46 @@
"""Transactional, checksummed migrations, serialized across concurrent starts."""
import hashlib
import os
from pathlib import Path
import psycopg
from messenger.settings import secret
def migrate(conninfo: str, directory: Path):
with psycopg.connect(conninfo, connect_timeout=5) as conn:
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
)
for path in sorted(directory.glob("*.sql")):
content = path.read_text()
checksum = hashlib.sha256(content.encode()).hexdigest()
current = conn.execute(
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
).fetchone()
if current:
if current[0] != checksum:
raise RuntimeError("Applied migration checksum changed: " + path.stem)
continue
conn.execute(content)
conn.execute(
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
(path.stem, checksum),
)
print("Applied migration " + path.stem)
def main():
migrate(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
)
if __name__ == "__main__":
main()
+60
View File
@@ -0,0 +1,60 @@
"""Explicit local operator actions, never exposed as a public HTTP endpoint."""
import argparse
import json
from messenger.pipeline import Pipeline
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=("failed", "replay", "new-version"))
parser.add_argument("attachment_id", nargs="?")
args = parser.parse_args()
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
try:
if args.action == "failed":
with db.connection() as conn:
rows = conn.execute(
"SELECT job_id,attachment_id,pipeline_version,attempt,error_code,"
"source_expired FROM attachment_jobs WHERE state='failed' "
"ORDER BY updated_at LIMIT 100"
).fetchall()
print(json.dumps(rows, default=str))
else:
if not args.attachment_id:
parser.error("attachment_id is required")
with db.connection() as conn:
row = conn.execute(
"SELECT j.source_key,a.object_key,a.storage_state FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id AND "
"a.pipeline_version=j.pipeline_version "
"WHERE a.id=%s",
(args.attachment_id,),
).fetchone()
if not row:
parser.error("Unknown attachment")
store = ObjectStore(settings)
try:
key = row["object_key"] if row["storage_state"] == "ready" else row["source_key"]
if not store.head(key):
parser.error("Source is no longer retained")
finally:
store.close()
job = Pipeline(db, settings).replay(
args.attachment_id, new_version=args.action == "new-version"
)
print(json.dumps({"job_id": job, "action": args.action}))
except ValueError as exc:
parser.error(str(exc))
finally:
db.close()
if __name__ == "__main__":
main()
+261
View File
@@ -0,0 +1,261 @@
"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
import os
import random
from datetime import UTC, datetime, timedelta
from psycopg.types.json import Jsonb
from messenger.domain import DomainError, new_id
def event(conn, job_id, name, **details):
conn.execute(
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
(job_id, name, Jsonb(details)),
)
def intent(conn, job_id, when=None):
conn.execute(
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
(job_id, when),
)
class Pipeline:
def __init__(self, repository, settings):
self.db, self.settings = repository, settings
def reserve(self, id, key):
with self.db.connection(write=True) as conn:
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
def accept(self, id, chat, owner, name, media, size, digest, key):
with self.db.connection(write=True) as conn:
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
).fetchone()
if (
not reservation
or reservation["state"] != "uploading"
or reservation["object_key"] != key
or reservation["expires_at"] <= datetime.now(UTC)
):
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
row = conn.execute(
"INSERT INTO attachments "
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"object_key,storage_state) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
(id, chat, owner, name, media, size, digest, key),
).fetchone()
job = new_id()
conn.execute(
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
"VALUES (%s,%s,1,%s)",
(job, id, key),
)
intent(conn, job)
event(conn, job, "accepted")
return row
def claim(self, envelope):
with self.db.connection(write=True) as conn:
# All writers lock attachment before job, including publication and operator replay.
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
).fetchone()
if not a:
return "unknown", None
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
(envelope.job_id,),
).fetchone()
if (
not j
or j["attachment_id"] != a["id"]
or j["pipeline_version"] != envelope.pipeline_version
):
return "unknown", None
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
event(conn, j["job_id"], "stale_delivery")
return "stale", None
if j["state"] == "ready":
event(conn, j["job_id"], "duplicate_ready")
return "ready", a
if j["state"] == "failed":
return "failed", None
now = datetime.now(UTC)
if j["state"] == "processing" or j["next_attempt_at"] > now:
return "busy", None
if envelope.attempt != j["attempt"] + 1:
# An older envelope cannot steal a scheduled retry or an explicit replay.
return "obsolete_attempt", None
fence = new_id()
conn.execute(
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
"lease_until=clock_timestamp()+(%s*interval '1 "
"second'),updated_at=clock_timestamp() "
"WHERE job_id=%s",
(fence, self.settings.lease_seconds, j["job_id"]),
)
conn.execute(
"UPDATE attachments SET storage_state='processing',"
"updated_at=clock_timestamp() WHERE id=%s",
(a["id"],),
)
event(
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
)
return "claimed", {
**a,
"job_id": str(j["job_id"]),
"source_key": j["source_key"],
"fence": fence,
"attempt": j["attempt"] + 1,
}
def publish(self, row, original_key, variants):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
).fetchone()
if not self.owns(a, j, row):
event(conn, row["job_id"], "publication_fenced")
return False
conn.execute(
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(original_key, Jsonb(variants), row["id"]),
)
conn.execute(
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
"updated_at=clock_timestamp() WHERE job_id=%s",
(row["job_id"],),
)
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
return True
@staticmethod
def owns(a, j, row):
return (
a
and j
and a["pipeline_version"] == row["pipeline_version"]
and j["state"] == "processing"
and str(j["fence"]) == row["fence"]
and j["lease_until"] > datetime.now(UTC)
)
def failure(self, row, code, permanent=False):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
).fetchone()
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
).fetchone()
if self.owns(a, j, row):
self._retry(conn, j, code, permanent)
def _retry(self, conn, j, code, permanent=False):
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
state = "failed" if failed else "queued"
if failed and not permanent:
code = "retry_exhausted"
# Full jitter; durable absolute time survives worker/relay restarts.
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
due = datetime.now(UTC) + timedelta(seconds=delay)
conn.execute(
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
"updated_at=clock_timestamp() WHERE job_id=%s",
(state, code, due, j["job_id"]),
)
conn.execute(
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
"WHERE id=%s",
(state, code if failed else None, j["attachment_id"]),
)
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
if not failed:
intent(conn, j["job_id"], due)
def recover(self):
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
with self.db.connection(write=True) as conn:
rows = conn.execute(
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
"FOR UPDATE OF a SKIP LOCKED"
).fetchall()
for row in rows:
j = conn.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
"UPDATE OF j",
(row["id"],),
).fetchone()
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
event(conn, j["job_id"], "lease_expired")
self._retry(conn, j, "worker_timeout")
def replay(self, attachment, *, new_version=False):
with self.db.connection(write=True) as conn:
a = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
).fetchone()
if not a:
raise ValueError("Unknown attachment")
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
"AND pipeline_version=%s FOR UPDATE",
(attachment, a["pipeline_version"]),
).fetchone()
if new_version:
if a["pipeline_version"] != 1:
raise ValueError("Only pipeline v1 -> v2 is implemented")
conn.execute(
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
"AND state IN ('queued','processing','failed')",
(j["job_id"],),
)
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
job = new_id()
conn.execute(
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
"VALUES (%s,%s,2,%s)",
(job, attachment, key),
)
conn.execute(
"UPDATE attachments SET "
"pipeline_version=2,storage_state='queued',variants='[]',"
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment,),
)
event(conn, job, "new_version", previous_job=str(j["job_id"]))
else:
if j["state"] != "failed" or j["source_expired"]:
raise ValueError("Replay requires failed job with retained source")
job = str(j["job_id"])
conn.execute(
"UPDATE attachment_jobs SET "
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
"job_id=%s",
(job,),
)
conn.execute(
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
"updated_at=clock_timestamp() WHERE id=%s",
(attachment,),
)
event(conn, job, "operator_replay")
intent(conn, job)
return job
+78
View File
@@ -0,0 +1,78 @@
"""Transactional outbox publisher and expired-lease watchdog, independently restartable."""
import signal
import time
from pathlib import Path
import redis
from messenger.broker import Broker, Job
from messenger.domain import DomainError
from messenger.faults import checkpoint
from messenger.pipeline import Pipeline, event
from messenger.repository import PostgresRepository, stamp
from messenger.settings import Settings
def publish_one(db, broker, settings):
with db.connection(write=True) as conn:
row = conn.execute(
"SELECT o.job_id,j.attachment_id,j.pipeline_version,j.attempt,j.created_at "
"FROM job_outbox o JOIN attachment_jobs j USING(job_id) "
"WHERE o.published_at IS NULL AND o.due_at<=clock_timestamp() AND j.state='queued' "
"ORDER BY o.due_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
).fetchone()
if not row:
return False
job = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(row["job_id"]),
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=stamp(row["created_at"]),
)
stream_id = broker.publish(job)
# Commit-on-return may be lost here; repeat sends the same logical job_id.
checkpoint(db, settings, row["job_id"], "after_publish")
conn.execute(
"UPDATE job_outbox SET published_at=clock_timestamp(),stream_id=%s WHERE job_id=%s",
(stream_id, row["job_id"]),
)
event(conn, row["job_id"], "published", stream_id=stream_id, attempt=job.attempt)
return True
def main():
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=4)
db.open()
broker = Broker(settings)
pipeline = Pipeline(db, settings)
running = True
def stop(*_):
nonlocal running
running = False
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while running:
try:
pipeline.recover()
sent = publish_one(db, broker, settings)
Path("/tmp/relay-heartbeat").touch()
if not sent:
time.sleep(0.2)
except redis.RedisError, DomainError:
# Keep the transaction uncommitted and retry. No credentials/SQL in logs.
time.sleep(0.5)
finally:
broker.close()
db.close()
if __name__ == "__main__":
main()
+339
View File
@@ -0,0 +1,339 @@
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
import hashlib
import json
from contextlib import contextmanager, nullcontext
from datetime import datetime
from typing import Protocol
import psycopg
from psycopg import errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
from messenger.domain import (
Attachment,
Chat,
DomainError,
HistoryKey,
Message,
User,
invalid,
new_id,
)
def stamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
def user_value(row) -> User:
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
def chat_value(row) -> Chat:
return Chat(
str(row["id"]),
row["title"],
tuple(str(x) for x in row["member_ids"]),
stamp(row["created_at"]),
)
def message_value(row) -> Message:
return Message(
str(row["id"]),
str(row["chat_id"]),
str(row["sender_id"]),
row["text"],
stamp(row["created_at"]),
tuple(str(x) for x in row.get("attachment_ids", ())),
)
def attachment_value(row) -> Attachment:
return Attachment(
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
row["filename"],
row["content_type"],
row["size_bytes"],
row["sha256"],
stamp(row["created_at"]),
stamp(row["updated_at"]),
row["storage_state"],
tuple(
{k: v for k, v in item.items() if k != "object_key"} for item in row.get("variants", [])
),
row.get("error_code"),
)
class Repository(Protocol):
def get_user(self, user_id: str) -> User | None: ...
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
def get_chat(self, chat_id: str) -> Chat | None: ...
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
def idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
key: str,
attachment_ids: tuple[str, ...] = (),
) -> Message: ...
def page_messages(
self, chat_id: str, after: HistoryKey | None, limit: int
) -> list[Message]: ...
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
class PostgresRepository:
def __init__(
self,
conninfo: str,
pool_size: int = 8,
idempotency_ttl: int = 86400,
read_autocommit: bool = True,
):
self.read_autocommit = read_autocommit
self.idempotency_ttl = idempotency_ttl
self.pool = ConnectionPool(
conninfo,
min_size=1,
max_size=pool_size,
timeout=1,
max_waiting=32,
open=False,
check=ConnectionPool.check_connection,
kwargs={
"row_factory": dict_row,
"autocommit": True,
"connect_timeout": 2,
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
},
)
def open(self):
self.pool.open(wait=True, timeout=15)
def close(self):
self.pool.close(timeout=5)
@contextmanager
def connection(self, *, write: bool = False):
try:
with self.pool.connection() as conn:
transaction = (
conn.transaction() if write or not self.read_autocommit else nullcontext()
)
with transaction:
yield conn
except (
psycopg.OperationalError,
psycopg.InterfaceError,
PoolTimeout,
TooManyRequests,
errors.QueryCanceled,
errors.LockNotAvailable,
):
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
def ready(self):
with self.connection() as conn:
row = conn.execute(
"SELECT count(*) AS n FROM schema_migrations "
"WHERE version IN "
"('001_initial','002_idempotency','003_attachments',"
"'004_pipeline','005_upload_handoff')"
).fetchone()
if row["n"] != 5:
raise DomainError(503, "unavailable", "Схема ещё не готова")
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO users (id,username,display_name,password_hash) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), username, display_name, password_hash),
).fetchone()
return user_value(row)
except errors.UniqueViolation:
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
def credentials(self, username: str):
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
return (user_value(row), row["password_hash"]) if row else None
def get_user(self, user_id: str) -> User | None:
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
return user_value(row) if row else None
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
[(row["id"], member, i) for i, member in enumerate(member_ids)],
)
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
except errors.ForeignKeyViolation:
raise invalid("Указан неизвестный участник") from None
def get_chat(self, chat_id: str) -> Chat | None:
with self.connection() as conn:
row = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
).fetchone()
return chat_value(row) if row else None
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (user_id, *after, limit) if after else (user_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
params,
).fetchall()
return [chat_value(row) for row in rows]
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
return message_value(row)
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (chat_id, *after, limit) if after else (chat_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s"
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
params,
).fetchall()
return [message_value(row) for row in rows]
def idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
key: str,
attachment_ids: tuple[str, ...] = (),
) -> Message:
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
payload = json.dumps(
{"text": text, "attachment_ids": list(attachment_ids)},
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
)
digest = hashlib.sha256(payload.encode()).digest()
with self.connection(write=True) as conn:
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
previous = conn.execute(
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
"JOIN messages m ON m.id=r.response_message_id "
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
(sender_id, chat_id, key),
).fetchone()
if previous:
if previous["payload_hash"] != digest:
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
return message_value(previous)
conn.execute(
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
"AND chat_id=%s AND idempotency_key=%s",
(sender_id, chat_id, key),
)
if attachment_ids:
rows = conn.execute(
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
(chat_id, list(attachment_ids)),
).fetchall()
if len(rows) != len(attachment_ids):
raise DomainError(404, "not_found", "Объект не найден")
if any(row["storage_state"] == "failed" for row in rows):
raise DomainError(409, "conflict", "Вложение пока недоступно")
message = message_value(
conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
)
for position, attachment in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
(message.id, chat_id, attachment, position),
)
message = Message(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
attachment_ids,
)
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
return message
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
conn.execute(
"INSERT INTO message_requests "
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
)
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO sessions (token_hash,user_id,expires_at) "
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
(token_hash, user_id, ttl),
).fetchone()
return stamp(row["expires_at"])
def active_session(self, token_hash: bytes):
with self.connection() as conn:
return conn.execute(
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
(token_hash,),
).fetchone()
def revoke_session(self, token_hash: bytes):
with self.connection(write=True) as conn:
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
def attachment(self, id):
with self.connection() as conn:
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
+120
View File
@@ -0,0 +1,120 @@
"""Strict lab 5 wire types; password never belongs to a response User."""
import re
from typing import Annotated
from uuid import UUID
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
def canonical_uuid(value: str) -> str:
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
raise ValueError("Ожидается UUID")
return str(UUID(value))
def nonblank(value: str) -> str:
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
raise ValueError("Ожидается непустая строка Unicode")
return value
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
class Input(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
class CreateUser(Input):
username: str = Field(min_length=3, max_length=32)
display_name: ShortText
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
@field_validator("username")
@classmethod
def valid_username(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Недопустимое имя пользователя")
return value
class CreateChat(Input):
title: ShortText
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
class CreateMessage(Input):
text: str = Field(max_length=4000)
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def valid_content(self):
if (
not self.text.strip()
and not self.attachment_ids
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
or len(set(self.attachment_ids)) != len(self.attachment_ids)
):
raise ValueError("Некорректное содержимое сообщения")
return self
class View(BaseModel):
model_config = ConfigDict(from_attributes=True, extra="forbid")
class UserView(View):
id: str
username: str
display_name: str
created_at: str
class SessionView(View):
token: str
token_type: str
expires_at: str
user: UserView
class ChatView(View):
id: str
title: str
member_ids: list[str]
created_at: str
class MessageView(View):
attachment_ids: list[str]
id: str
chat_id: str
sender_id: str
text: str
created_at: str
class ChatPage(View):
items: list[ChatView]
next_cursor: str | None
class MessagePage(View):
items: list[MessageView]
next_cursor: str | None
class AttachmentView(View):
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list
error_code: str | None
created_at: str
updated_at: str
+52
View File
@@ -0,0 +1,52 @@
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
from dataclasses import dataclass
from messenger.cursor import decode_cursor, encode_cursor
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
from messenger.repository import Repository
@dataclass(frozen=True)
class Page[T]:
items: list[T]
next_cursor: str | None
class Messenger:
def __init__(self, repository: Repository):
self.repository = repository
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
return self.repository.add_chat(title, tuple(member_ids))
def chat(self, actor: User, chat_id: str) -> Chat:
chat = self.repository.get_chat(chat_id)
if chat is None or actor.id not in chat.member_ids:
raise not_found()
return chat
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
scope = "chats:" + actor.id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
def send(
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
) -> Message:
self.chat(actor, chat_id)
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
self.chat(actor, chat_id)
scope = "messages:" + chat_id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
@staticmethod
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
visible = items[:limit]
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
return Page(visible, cursor)
+85
View File
@@ -0,0 +1,85 @@
"""Explicit configuration; credentials are read from untracked secret files."""
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str, default: str, optional=False) -> str:
path = Path(os.getenv(name, default))
return path.read_text().strip() if path.exists() or not optional else ""
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 8
idempotency_ttl: int = 86400
read_autocommit: bool = True
s3_endpoint: str = "http://127.0.0.1:59005"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key: str = field(default="", repr=False)
s3_secret_key: str = field(default="", repr=False)
max_upload_bytes: int = 10485760
max_image_pixels: int = 40000000
upload_slots: int = 1
broker_url: str = field(default="redis://broker:6379/0", repr=False)
worker_concurrency: int = 2
job_max_attempts: int = 5
job_timeout: int = 20
staging_retention: int = 86400
fault_injection: bool = False
@property
def lease_seconds(self):
return self.job_timeout + 3
@classmethod
def from_env(cls):
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
if idempotency_ttl < 86400:
raise ValueError("Idempotency retention must be at least 24 hours")
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
raise ValueError("Invalid session lifetime or connection pool size")
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
raise ValueError("Invalid upload limits")
concurrency = int(os.getenv("WORKER_CONCURRENCY", "2"))
attempts = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
timeout = int(os.getenv("JOB_TIMEOUT_SECONDS", "20"))
retention = int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))
if not 1 <= concurrency <= 4 or not 1 <= attempts <= 10 or not 2 <= timeout <= 120:
raise ValueError("Invalid worker limits")
if retention < 300:
raise ValueError("Staging retention must be at least 300 seconds")
return cls(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
ttl,
size,
idempotency_ttl,
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
os.getenv("S3_ENDPOINT", "http://s3:9000"),
os.getenv("S3_REGION", "us-east-1"),
os.getenv("S3_BUCKET", "messenger-attachments"),
secret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id", optional=True),
secret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key", optional=True),
maximum,
pixels,
broker_url=secret("BROKER_URL_FILE", "/run/secrets/broker_url")
if os.path.exists(os.getenv("BROKER_URL_FILE", "/run/secrets/broker_url"))
else "redis://broker:6379/0",
worker_concurrency=concurrency,
job_max_attempts=attempts,
job_timeout=timeout,
staging_retention=retention,
fault_injection=os.getenv("ENABLE_FAULT_INJECTION", "0") == "1",
)
+74
View File
@@ -0,0 +1,74 @@
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from messenger.domain import DomainError
def unavailable():
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
class ObjectStore:
def __init__(self, settings):
self.bucket = settings.s3_bucket
self.client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key,
aws_secret_access_key=settings.s3_secret_key,
config=Config(
connect_timeout=0.5,
read_timeout=2,
max_pool_connections=8,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except BotoCoreError, ClientError:
raise unavailable() from None
def put(self, key, file, size, media, digest):
try:
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=file,
ContentLength=size,
ContentType=media,
Metadata={"sha256": digest},
)
except BotoCoreError, ClientError:
raise unavailable() from None
def head(self, key):
try:
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if str(exc.response.get("Error", {}).get("Code")) in ("404", "NoSuchKey", "NotFound"):
return None
raise unavailable() from None
except BotoCoreError:
raise unavailable() from None
def download(self, row):
try:
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
if response["ContentLength"] != row["size_bytes"]:
response["Body"].close()
raise unavailable()
return response["Body"]
except BotoCoreError, ClientError:
raise unavailable() from None
def close(self):
self.client.close()
+200
View File
@@ -0,0 +1,200 @@
"""Bounded supervisor: one disposable OS process per job, hard wall-clock and RAM limits."""
import hashlib
import io
import multiprocessing
import resource
import signal
import tempfile
import time
from pathlib import Path
from uuid import uuid4
import redis
from pydantic import ValidationError
from messenger.attachments import image_type
from messenger.broker import GROUP, STREAM, Broker, Job
from messenger.domain import DomainError
from messenger.faults import PermanentFailure, checkpoint
from messenger.images import variants
from messenger.pipeline import Pipeline, event
from messenger.repository import PostgresRepository
from messenger.settings import Settings
from messenger.storage import ObjectStore
def verify_effect(store, row):
records = [row, *row["variants"]]
if len(records) != 4 or {v["name"] for v in row["variants"]} != {
"thumbnail",
"crop",
"optimized",
}:
raise DomainError(503, "unavailable", "Incomplete result")
for item in records:
head = store.head(item["object_key"])
if (
not head
or head["ContentLength"] != item["size_bytes"]
or head["Metadata"].get("sha256") != item["sha256"]
):
raise DomainError(503, "unavailable", "Incomplete result")
def handle(db, broker, store, settings, id, payload):
try:
job = Job.model_validate_json(payload)
except ValidationError:
broker.quarantine(id, "invalid_envelope")
return
pipeline = Pipeline(db, settings)
outcome, row = pipeline.claim(job)
if outcome == "busy":
return # Leave pending. Watchdog recovers the DB lease and schedules a retry.
if outcome == "unknown":
broker.quarantine(id, "unknown_job")
return
if outcome == "ready":
verify_effect(store, row)
broker.ack(id)
return
if outcome != "claimed":
broker.ack(id) # Failed registry / obsolete version / durable retry owns this work.
return
try:
checkpoint(db, settings, job.job_id, "before_effect")
with tempfile.TemporaryFile() as original:
body = store.download({**row, "object_key": row["source_key"]})
size, digest = 0, hashlib.sha256()
try:
while chunk := body.read(65536):
size += len(chunk)
if size > settings.max_upload_bytes:
raise PermanentFailure
digest.update(chunk)
original.write(chunk)
finally:
body.close()
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
raise PermanentFailure
image_type(original, settings.max_image_pixels)
prefix = f"objects/{row['id']}/v{row['pipeline_version']}/"
original_key = prefix + "original"
store.put(original_key, original, size, row["content_type"], digest.hexdigest())
checkpoint(db, settings, job.job_id, "after_effect")
records = []
for name, data, (width, height), media in variants(original, row["pipeline_version"]):
checksum = hashlib.sha256(data).hexdigest()
key = prefix + name
store.put(key, io.BytesIO(data), len(data), media, checksum)
records.append(
{
"name": name,
"content_type": media,
"size_bytes": len(data),
"sha256": checksum,
"width": width,
"height": height,
"object_key": key,
}
)
with db.connection(write=True) as conn:
event(conn, job.job_id, "objects_written", version=row["pipeline_version"])
committed = pipeline.publish(row, original_key, records)
if committed:
checkpoint(db, settings, job.job_id, "after_commit")
broker.ack(id)
except PermanentFailure:
pipeline.failure(row, "invalid_image", permanent=True)
broker.ack(id)
except MemoryError:
pipeline.failure(row, "resource_limit", permanent=True)
broker.ack(id)
except DomainError as exc:
pipeline.failure(
row,
"invalid_image" if exc.status in (413, 415) else "storage_unavailable",
permanent=exc.status in (413, 415),
)
broker.ack(id)
except redis.RedisError:
# Result may already be committed; retain the pending entry for a verified no-op.
raise
except Exception:
pipeline.failure(row, "processing_failed")
broker.ack(id)
def child(settings, id, payload):
# Linux hard limit includes Pillow buffers and native allocations. Supervisor kills hangs.
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
db = PostgresRepository(settings.database_url, pool_size=2)
db.open()
broker, store = Broker(settings), ObjectStore(settings)
try:
handle(db, broker, store, settings, id, payload)
except redis.RedisError, DomainError:
pass # Do not emit secret-bearing driver tracebacks; leave pending / lease intact.
finally:
store.close()
broker.close()
db.close()
def main():
settings = Settings.from_env()
broker = Broker(settings)
consumer, cursor = uuid4().hex, "0-0"
context = multiprocessing.get_context("spawn")
active, running = [], True
def stop(*_):
nonlocal running
running = False
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while running:
for process, deadline in active[:]:
if not process.is_alive() or time.monotonic() > deadline:
if process.is_alive():
process.kill()
process.join()
active.remove((process, deadline))
process.close()
Path("/tmp/worker-heartbeat").touch()
if len(active) >= settings.worker_concurrency:
time.sleep(0.05)
continue
try:
broker.group()
reclaimed = broker.client.xautoclaim(
STREAM, GROUP, consumer, settings.lease_seconds * 1000, cursor, count=1
)
cursor, entries = reclaimed[0], reclaimed[1]
if not entries:
result = broker.client.xreadgroup(
GROUP, consumer, {STREAM: ">"}, count=1, block=200
)
entries = result[0][1] if result else []
if entries:
id, fields = entries[0]
process = context.Process(
target=child, args=(settings, id, fields.get("job", ""))
)
process.start()
active.append((process, time.monotonic() + settings.job_timeout))
except redis.RedisError:
time.sleep(0.5)
finally:
for process, _ in active:
if process.is_alive():
process.kill()
process.join()
broker.close()
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
import base64
from pathlib import Path
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from psycopg import sql
from psycopg.conninfo import make_conninfo
from messenger.app import create_app
from messenger.migrate import migrate
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
PASSWORD = "Учебный:пароль😀-12345"
@pytest.fixture(scope="session")
def settings():
password = (ROOT / "secrets/postgres_password").read_text().strip()
admin = make_conninfo(
host="127.0.0.1", port=55437, user="messenger", password=password, dbname="messenger"
)
database = "student_tests_" + uuid4().hex
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
config = Settings(
make_conninfo(admin, dbname=database),
"redis://127.0.0.1:56384/1",
broker_url="redis://127.0.0.1:56385/1",
s3_bucket="messenger-attachments-tests",
s3_access_key=(ROOT / "secrets/s3_test_access_key_id").read_text().strip(),
s3_secret_key=(ROOT / "secrets/s3_test_secret_access_key").read_text().strip(),
)
migrate(config.database_url, ROOT / "migrations")
with pytest.MonkeyPatch.context() as environment:
environment.setenv("S3_ENDPOINT", config.s3_endpoint)
environment.setenv("S3_BUCKET", config.s3_bucket)
environment.setenv("S3_ACCESS_KEY_ID_FILE", str(ROOT / "secrets/s3_test_access_key_id"))
environment.setenv(
"S3_SECRET_ACCESS_KEY_FILE", str(ROOT / "secrets/s3_test_secret_access_key")
)
yield config
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
@pytest.fixture
def client(settings):
with psycopg.connect(settings.database_url) as conn:
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
client.app.state.cache.client.flushdb()
yield client
def basic(username, password=PASSWORD):
return {
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
}
@pytest.fixture
def actors(client):
result = []
for name in ("alice", "bob", "eve"):
r = client.post(
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
)
assert r.status_code == 201
session = client.post("/api/v1/auth/sessions", headers=basic(name))
assert session.status_code == 201
result.append({**r.json(), "token": session.json()["token"]})
return result
@pytest.fixture
def room(client, actors):
response = client.post(
"/api/v1/chats",
headers={"Authorization": "Bearer " + actors[0]["token"]},
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
)
assert response.status_code == 201
return response.json()
+299
View File
@@ -0,0 +1,299 @@
import hashlib
import json
import re
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import psycopg
import pytest
from conftest import PASSWORD, ROOT, basic
from fastapi.testclient import TestClient
from messenger.app import create_app
from messenger.auth import basic_credentials
from messenger.migrate import migrate
def headers(actor):
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
def error(response, status, code):
assert response.status_code == status, response.text
body = response.json()["error"]
assert set(body) == {"code", "message", "request_id"}
assert body["code"] == code
assert body["request_id"] == response.headers["X-Request-Id"]
assert len(response.headers.get_list("X-Request-Id")) == 1
if status == 401:
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
def test_malformed_json(client, body):
error(
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
400,
"invalid_request",
)
@pytest.mark.parametrize(
"change",
[
{"username": "ab"},
{"username": "a" * 33},
{"username": "Upper"},
{"username": "кириллица"},
{"username": "valid\n"},
{"username": 123},
{"username": True},
{"display_name": 123},
{"display_name": None},
{"display_name": " \n\t"},
{"display_name": "😀" * 101},
{"display_name": "\ud800"},
{"password": "short"},
{"password": "😀" * 129},
{"password": 123},
{"password": " " * 12},
{"password": "\ud800" * 12},
{"extra": True},
],
)
def test_strict_registration(client, change):
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
response = client.post(
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
)
error(response, 400, "invalid_request")
def test_unicode_password_hash_and_authentication(client, settings, actors):
for u in actors:
assert "password" not in u and "password_hash" not in u
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
with psycopg.connect(settings.database_url) as conn:
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
0
]
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
assert (
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
== 32
)
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
error(bad, 401, "unauthorized")
error(absent, 401, "unauthorized")
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
for value in (
{},
basic("alice"),
{"X-User-Id": actors[0]["id"]},
{"Authorization": "Bearer unknown"},
):
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
actual = client.get(
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
).json()
assert actual["id"] == actors[0]["id"]
error(
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
400,
"invalid_request",
)
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
def test_invalid_basic(value):
with pytest.raises(Exception) as caught:
basic_credentials(value)
assert caught.value.status == 401
def test_registration_race(client, settings):
def register(_):
return client.post(
"/api/v1/users",
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
).status_code
with ThreadPoolExecutor(max_workers=2) as pool:
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
path = "/api/v1/chats/" + room["id"]
for suffix in ("", "/messages"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
error(
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
)
with psycopg.connect(settings.database_url) as conn:
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
error(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "rollback", "member_ids": members},
),
400,
"invalid_request",
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
for body in (
{"text": ""},
{"text": " \t"},
{"text": "😀" * 4001},
{"text": 123},
{"text": None},
{"text": "ok", "sender_id": actors[1]["id"]},
):
error(
client.post(path + "/messages", headers=headers(actors[0]), json=body),
400,
"invalid_request",
)
for query in (
"limit=0",
"limit=101",
"limit=true",
"limit=1.0",
"limit=1&limit=2",
"cursor=",
"cursor=bad!",
"cursor=e30",
"cursor=x&cursor=y",
):
error(
client.get(path + "/messages?" + query, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
def send(i):
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
assert r.status_code == 201
return r.json()
with ThreadPoolExecutor(max_workers=10) as pool:
sent = list(pool.map(send, range(100)))
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
)
for m in sent:
m["created_at"] = "2026-10-17T12:00:00.000000Z"
found, cursor = [], None
while True:
query = {"limit": 17 if cursor else 1}
if cursor:
query["cursor"] = cursor
response = client.get(path, params=query, headers=headers(actors[1]))
assert response.status_code == 200
page = response.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
assert len(found) == len({x["id"] for x in found}) == 100
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
error(
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
user = actors[0]
url = "/api/v1/users/me"
assert client.get(url, headers=headers(user)).status_code == 200
repository = client.app.state.repository
original = repository.get_user
def must_be_cached(*args):
raise AssertionError("cache hit should avoid fetching immutable user again")
monkeypatch.setattr(repository, "get_user", must_be_cached)
assert client.get(url, headers=headers(user)).status_code == 200
monkeypatch.setattr(repository, "get_user", original)
cache = client.app.state.cache.client
digest = hashlib.sha256(user["token"].encode()).digest()
key = client.app.state.cache.key(digest)
assert 0 < cache.ttl(key) <= 3600
stale = cache.get(key)
cache.flushdb()
assert client.get(url, headers=headers(user)).status_code == 200
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
error(client.get(url, headers=headers(user)), 401, "unauthorized")
error(
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
)
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
database_file.write_text(settings.database_url)
cache_file.write_text(settings.cache_url)
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
with TestClient(create_app()) as short:
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
assert response.status_code == 201
token = {"Authorization": "Bearer " + response.json()["token"]}
assert short.get("/api/v1/users/me", headers=token).status_code == 200
with TestClient(create_app(settings)) as restarted:
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
time.sleep(2.1)
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
migrate(settings.database_url, ROOT / "migrations")
modified = tmp_path / "001_initial.sql"
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
with pytest.raises(RuntimeError, match="checksum changed"):
migrate(settings.database_url, tmp_path)
def test_logs_and_safe_errors(client, monkeypatch, caplog):
canary = "PRIVATE_BODY_CANARY_123"
with caplog.at_level("INFO", logger="messenger"):
r = client.post(
"/api/v1/users",
json={"username": "logging", "display_name": canary, "password": PASSWORD},
)
assert r.status_code == 201
assert canary not in caplog.text and PASSWORD not in caplog.text
def broken(*args):
raise RuntimeError(canary)
monkeypatch.setattr(client.app.state.authentication, "register", broken)
response = client.post(
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
)
error(response, 500, "internal_error")
assert canary not in response.text and canary not in caplog.text
error(
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
415,
"unsupported_media_type",
)
+137
View File
@@ -0,0 +1,137 @@
import json
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from test_api import error, headers
from messenger.app import create_app
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
def test_message_key_required(client, actors, room, key):
h = {"Authorization": "Bearer " + actors[0]["token"]}
if key is not None:
h["Idempotency-Key"] = key
error(
client.post(
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
),
400,
"invalid_request",
)
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
path = f"/api/v1/chats/{room['id']}/messages"
key = "Scope.Key:123"
h = {**headers(actors[0]), "Idempotency-Key": key}
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
assert first.status_code == 201
repeat = client.post(
path,
headers={**h, "Content-Type": "application/json"},
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
)
assert repeat.status_code == 201 and repeat.json() == first.json()
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
outsider = {**headers(actors[2]), "Idempotency-Key": key}
error(
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
)
other_user = {**headers(actors[1]), "Idempotency-Key": key}
own = client.post(path, headers=other_user, json={"text": "another author"})
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
chat = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "another", "member_ids": [actors[0]["id"]]},
).json()
other_chat = client.post(
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
)
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
with TestClient(create_app(settings)) as second:
assert second.app.state.instance_id != client.app.state.instance_id
def send(i):
c = client if i % 2 else second
r = c.post(path, headers=h, json={"text": "same concurrent body"})
assert r.status_code == 201
return r.json(), r.headers["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({item[0]["id"] for item in results}) == 1
assert len({item[1] for item in results}) == 2
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
repository = client.app.state.repository
original = repository._save_request
def fail_after_message_insert(*args):
raise RuntimeError("injected between message and result")
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
error(
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
monkeypatch.setattr(repository, "_save_request", original)
result = client.post(path, headers=h, json={"text": "retry after rollback"})
assert result.status_code == 201
assert (
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
)
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
first = client.post(path, headers=h, json={"text": "old"}).json()
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
"expires_at=clock_timestamp()-interval '1 day'"
)
second = client.post(path, headers=h, json={"text": "new"})
assert second.status_code == 201 and second.json()["id"] != first["id"]
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
@pytest.mark.parametrize("optimized", [True, False])
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
c.get("/api/v1/users/me", headers=headers(actors[0]))
pool = c.app.state.repository.pool
before = pool.get_stats().get("requests_num", 0)
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
assert response.status_code == 200 and response.json()["items"] == []
assert pool.get_stats()["requests_num"] - before == 3
def test_instance_header_covers_health_errors_and_success(client):
values = [
client.get("/health/live"),
client.get("/health/ready"),
client.get("/api/v1/users/me"),
client.get("/not-a-route"),
]
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
+357
View File
@@ -0,0 +1,357 @@
import hashlib
import io
import json
import struct
import zlib
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from urllib.parse import quote
from uuid import uuid4
import jsonschema
import pytest
from conftest import ROOT
from PIL import Image
from test_api import error, headers
from messenger.broker import GROUP, STREAM, Broker, Job
from messenger.cleanup import collect
from messenger.domain import DomainError
from messenger.images import variants
from messenger.pipeline import Pipeline
from messenger.relay import publish_one
from messenger.worker import handle
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
def upload(client, actor, room, data=PNG, media="image/png", name="Фото.png"):
return client.post(
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
headers={**headers(actor), "Content-Type": media},
content=data,
)
@pytest.fixture
def broker(settings):
b = Broker(settings)
assert settings.broker_url.endswith("/1")
b.client.flushdb()
b.group()
yield b
b.client.flushdb()
b.close()
def process(client, settings, broker):
db = client.app.state.repository
assert publish_one(db, broker, settings)
id, fields = broker.client.xreadgroup(GROUP, "unit", {STREAM: ">"}, count=1)[0][1][0]
jsonschema.validate(
json.loads(fields["job"]),
json.loads((ROOT / "contracts/jobs.schema.json").read_text()),
format_checker=jsonschema.FormatChecker(),
)
handle(db, broker, client.app.state.attachments.store, settings, id, fields["job"])
def test_queued_message_ready_and_all_variants(client, actors, room, settings, broker):
r = upload(client, actors[0], room)
assert r.status_code == 202 and r.json()["status"] == "queued"
a = r.json()
assert not a["variants"] and a["sha256"] == hashlib.sha256(PNG).hexdigest()
path = "/api/v1/attachments/" + a["id"]
error(client.get(path + "/content", headers=headers(actors[0])), 409, "conflict")
key = uuid4().hex
msg = client.post(
f"/api/v1/chats/{room['id']}/messages",
json={"text": "", "attachment_ids": [a["id"]]},
headers={**headers(actors[0]), "Idempotency-Key": key},
)
assert msg.status_code == 201
for suffix in ("", "/content"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get(path + suffix), 401, "unauthorized")
process(client, settings, broker)
ready = client.get(path, headers=headers(actors[1])).json()
assert ready["status"] == "ready" and ready["error_code"] is None
assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}
assert client.get(path + "/content", headers=headers(actors[1])).content == PNG
for v in ready["variants"]:
content = client.get(path + "/content?variant=" + v["name"], headers=headers(actors[1]))
assert content.status_code == 200 and content.headers["content-type"] == v["content_type"]
assert len(content.content) == v["size_bytes"]
assert hashlib.sha256(content.content).hexdigest() == v["sha256"]
with Image.open(io.BytesIO(content.content)) as image:
assert image.size == (v["width"], v["height"])
assert not image.getexif()
with client.app.state.repository.connection() as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
).fetchone()
job = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=a["id"],
pipeline_version=1,
attempt=1,
created_at=a["created_at"],
)
sid = broker.publish(job)
handle(
client.app.state.repository,
broker,
client.app.state.attachments.store,
settings,
sid,
job.model_dump_json(exclude_none=True),
)
assert client.get(path, headers=headers(actors[0])).json() == ready
assert broker.client.xlen(STREAM) == 0
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
def test_bad_filename(client, actors, room, name):
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
@pytest.mark.parametrize(
"data,media",
[
(b"", "image/png"),
(b"bad", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "text/plain"),
],
)
def test_bad_image(client, actors, room, data, media):
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
def test_upload_limits_and_pixel_bomb(client, actors, room):
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
error(
upload(client, actors[0], room, (b"x" * 1048576 for _ in range(11))),
413,
"payload_too_large",
)
data = bytearray(PNG)
data[16:24] = struct.pack(">II", 10000, 10000)
data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xFFFFFFFF)
error(upload(client, actors[0], room, bytes(data)), 413, "payload_too_large")
@pytest.mark.parametrize("size", [(320, 200), (17, 11), (1, 400), (400, 1), (1281, 713)])
def test_transform_dimensions_alpha_and_exif(size):
raw = io.BytesIO()
Image.new("RGBA", size, (1, 2, 3, 0)).save(raw, "PNG")
results = {name: (data, dims, media) for name, data, dims, media in variants(raw, 1)}
scale = max(size)
assert results["thumbnail"][1] == tuple(max(1, n * min(256, scale) // scale) for n in size)
assert results["crop"][1] == (128, 128)
assert max(results["optimized"][1]) <= 1280
with Image.open(io.BytesIO(results["thumbnail"][0])) as image:
assert image.getpixel((0, 0)) == (255, 255, 255)
def test_exif_orientation_and_original_unchanged():
raw = io.BytesIO()
exif = Image.Exif()
exif[274] = 6
Image.new("RGB", (320, 200), "red").save(raw, "JPEG", exif=exif)
before = raw.getvalue()
result = list(variants(raw, 1))
assert result[0][2] == (160, 256) and raw.getvalue() == before
for _, data, _, _ in result:
with Image.open(io.BytesIO(data)) as image:
assert not image.getexif()
def test_failed_retry_replay_and_ready_replay_rejected(
client, actors, room, settings, broker, monkeypatch
):
a = upload(client, actors[0], room).json()
store, db = client.app.state.attachments.store, client.app.state.repository
original = store.put
monkeypatch.setattr(
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
)
process(client, replace(settings, job_max_attempts=1), broker)
path = "/api/v1/attachments/" + a["id"]
assert client.get(path, headers=headers(actors[0])).json()["error_code"] == "retry_exhausted"
error(
client.post(
f"/api/v1/chats/{room['id']}/messages",
json={"text": "", "attachment_ids": [a["id"]]},
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
),
409,
"conflict",
)
Pipeline(db, settings).replay(a["id"])
monkeypatch.setattr(store, "put", original)
process(client, settings, broker)
assert client.get(path, headers=headers(actors[0])).json()["status"] == "ready"
with pytest.raises(ValueError):
Pipeline(db, settings).replay(a["id"])
def test_invalid_envelope_quarantined(client, settings, broker):
sid = broker.client.xadd(STREAM, {"job": '{"url":"http://untrusted.invalid/"}'})
handle(
client.app.state.repository,
broker,
client.app.state.attachments.store,
settings,
sid,
'{"url":"http://untrusted.invalid/"}',
)
assert broker.client.xlen("invalid-jobs") == 1
assert "url" not in json.dumps(broker.client.xrange("invalid-jobs"))
def test_staging_failure_has_no_accepted_job(client, actors, room, monkeypatch):
store = client.app.state.attachments.store
monkeypatch.setattr(
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
)
error(upload(client, actors[0], room), 503, "unavailable")
with client.app.state.repository.connection() as conn:
assert conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] == 0
def test_gc_minimum_retention(client):
with pytest.raises(ValueError):
collect(client.app.state.repository, client.app.state.attachments.store, 0, True)
def test_gc_fences_paused_upload_before_job_commit(client, actors, room, settings, monkeypatch):
db, store = client.app.state.repository, client.app.state.attachments.store
p = Pipeline(db, settings)
id = str(uuid4())
key = "staging/" + id
p.reserve(id, key)
store.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
# Simulated S3 object age; actual private test-bucket deletion and DB locks are real.
old = datetime.now(UTC) - timedelta(seconds=301)
head = store.head(key)
head["LastModified"] = old
objects = [{"Key": key, "ETag": head["ETag"], "Size": len(PNG), "LastModified": old}]
monkeypatch.setattr(store, "head", lambda _: head)
monkeypatch.setattr(
store.client,
"get_paginator",
lambda _: SimpleNamespace(paginate=lambda **_: [{"Contents": objects}]),
)
assert collect(db, store, 300, True)["protected"] == 1
with db.connection(write=True) as conn:
conn.execute(
"UPDATE upload_reservations SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE id=%s",
(id,),
)
assert collect(db, store, 300, False)["eligible"] == 1
assert collect(db, store, 300, True)["deleted"] == 1
with pytest.raises(DomainError) as expired:
p.accept(
id,
room["id"],
actors[0]["id"],
"paused.png",
"image/png",
len(PNG),
hashlib.sha256(PNG).hexdigest(),
key,
)
assert expired.value.status == 503 and db.attachment(id) is None
def test_version_fence_rejects_old_writer(client, actors, room, settings, broker):
a = upload(client, actors[0], room).json()
db, store = client.app.state.repository, client.app.state.attachments.store
with db.connection() as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
).fetchone()
envelope = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=a["id"],
pipeline_version=1,
attempt=1,
created_at=a["created_at"],
)
pipeline = Pipeline(db, settings)
outcome, old = pipeline.claim(envelope)
assert outcome == "claimed"
pipeline.replay(a["id"], new_version=True)
process(client, settings, broker)
current = db.attachment(a["id"])
assert current["pipeline_version"] == 2 and current["storage_state"] == "ready"
assert not pipeline.publish(old, "objects/old-result", [])
handle(
db,
broker,
store,
settings,
broker.publish(envelope),
envelope.model_dump_json(exclude_none=True),
)
assert db.attachment(a["id"])["object_key"] == current["object_key"]
def test_foreign_chat_attachment_is_not_linkable(client, actors, room):
a = upload(client, actors[0], room).json()
other = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "other", "member_ids": [actors[0]["id"]]},
).json()
error(
client.post(
f"/api/v1/chats/{other['id']}/messages",
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
),
404,
"not_found",
)
@pytest.mark.parametrize(
"patch",
[
{"schema_version": True},
{"traceparent": None},
{"tracestate": None},
{"url": "https://untrusted.invalid/"},
{"job_id": "not-a-uuid"},
{"pipeline_version": 0},
{"attempt": 0},
{"created_at": "2026-10-17T12:00:00"},
{"traceparent": "invalid"},
{"type": "download.anything"},
],
)
def test_job_schema_rejects_invalid_carrier(client, settings, broker, patch):
payload = {
"schema_version": 1,
"type": "attachment.process.v1",
"job_id": str(uuid4()),
"attachment_id": str(uuid4()),
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-10-17T12:00:00Z",
**patch,
}
data = json.dumps(payload)
sid = broker.client.xadd(STREAM, {"job": data})
handle(
client.app.state.repository, broker, client.app.state.attachments.store, settings, sid, data
)
assert broker.client.xrange("invalid-jobs")[0][1]["reason"] == "invalid_envelope"
Generated
+1075
View File
File diff suppressed because it is too large Load Diff