ЛР 5 — БВТ2301 — Маслов — на 5 #70
@@ -18,3 +18,20 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
**/__pycache__
|
||||
**/*.py[cod]
|
||||
.work
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
tests
|
||||
student-tests
|
||||
research
|
||||
scripts
|
||||
contracts
|
||||
*.md
|
||||
Makefile
|
||||
compose.yaml
|
||||
|
||||
.work*
|
||||
|
||||
student-unit
|
||||
|
||||
+11
-4
@@ -13,13 +13,20 @@ API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_BUCKET=messenger
|
||||
S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
PUBLIC_PORT=8085
|
||||
API_CPUS=0.5
|
||||
API_MEMORY=768m
|
||||
DB_POOL_SIZE=10
|
||||
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_TIMEOUT_SECONDS=20
|
||||
JOB_LEASE_SECONDS=23
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
## Лабораторная 5
|
||||
## Лабораторная5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Маслов, БВТ2301. Цель5, трек2: идемпотентность versioned image pipeline.
|
||||
Собственная ЛР4 перенесена с объявленным переходом upload201→202queued.
|
||||
Реализованы durable private S3 staging/reservation, atomic PG job+outbox,
|
||||
persistent Valkey Streams/AOF/WAITAOF, relay/watchdog, отдельный bounded worker,
|
||||
4variants, version/fence/live-lease CAS, finite retry/jitter/failed registry,
|
||||
audited replay/newversion и safe retention cleanup.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Контракт и публичные tests сохранены. Запуск, схема, fault methodology,
|
||||
результаты и ответы защиты — REPORT.md/SCHEMA.md.112HTTP и22unit прошли;
|
||||
полный clean-clone-final PASS:19fault cases, cluster/rolling/persistence,
|
||||
real-age cleanup,6runtime source/OCI checks и fsck. Строгое pending-before-SIGKILL
|
||||
окно COMMIT→ACK подтверждено дважды. Промежуточные failed runs сохранены.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Ограничения: одинhost/SPOFs, нет production TLS/backup/global admission quota;
|
||||
HEAD не periodic byte audit; at-least-once с идемпотентным эффектом,
|
||||
не exactly-once delivery; удалённый CI не подтверждён.
|
||||
При разработке кода, проверок и отчёта использована помощь ИИ. Источники —
|
||||
teacher contracts, собственная ЛР4, pinned libraries и официальные Valkey/Pillow docs.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
name: Application integration
|
||||
on: [push, pull_request, workflow_dispatch]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
COMPOSE_FILE: compose.yaml:compose.test.yaml
|
||||
NO_PROXY: localhost,127.0.0.1
|
||||
no_proxy: localhost,127.0.0.1
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: 0.13.7
|
||||
- run: uv sync --locked
|
||||
- run: make bootstrap check validate lint unit PYTHON=.venv/bin/python
|
||||
- run: VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
- run: make test test-student PYTHON=.venv/bin/python BASE_URL=http://127.0.0.1:8085
|
||||
- run: uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-ci-seed.json
|
||||
- run: uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-ci.json
|
||||
- run: uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-ci-seed.json --output evidence/gc-ci.json
|
||||
- run: uv run --locked python -m scripts.verify_cluster --output evidence/cluster-ci.json
|
||||
- if: always()
|
||||
run: docker compose down --volumes --remove-orphans
|
||||
@@ -30,3 +30,8 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
.work/
|
||||
.work-*.txt
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
|
||||
COPY --from=uv /uv /usr/local/bin/uv
|
||||
WORKDIR /app
|
||||
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
|
||||
ARG VCS_REF=unknown
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
WORKDIR /app
|
||||
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY src /app/src
|
||||
COPY migrations /app/migrations
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "-m", "src"]
|
||||
@@ -28,3 +28,26 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
UV ?= uv
|
||||
BASE_URL = http://127.0.0.1:8085
|
||||
.PHONY: bootstrap up down lint test-student verify
|
||||
bootstrap:
|
||||
$(PYTHON) scripts/bootstrap.py
|
||||
up: bootstrap
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
down:
|
||||
docker compose down
|
||||
lint:
|
||||
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
|
||||
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
|
||||
verify:
|
||||
$(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json
|
||||
|
||||
.PHONY: unit reconcile
|
||||
unit:
|
||||
PYTHONPATH=. $(UV) run --locked pytest student-unit -q
|
||||
reconcile:
|
||||
docker compose run --rm --no-deps reconciler python -m src.reconcile
|
||||
|
||||
@@ -1,37 +1,412 @@
|
||||
# Отчёт — лабораторная 5
|
||||
# Лабораторная 5 — Маслов, БВТ2301
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Цель: **5, трек 2 — идемпотентность versioned pipeline**. Собственная ЛР4
|
||||
перенесена в независимую историю текущего teacher skeleton; обязательные
|
||||
contracts/tests/check/validate сохранены. API принимает исходник, а отдельный
|
||||
worker публикует original/thumbnail/crop/optimized. Исследование проверяет
|
||||
дубли, reorder и реальные SIGKILL, а не моделирует их ответами mock.
|
||||
|
||||
## Запуск из чистого clone
|
||||
Полный повтор из чистого clone — **PASS**, проверенный runtime source
|
||||
`442163b0f0a549ef065cb13ee39b5ca2f8580c18`. Прошли112HTTP,22unit, public smoke,
|
||||
19fault cases, cluster/rolling/persistence и cleanup с настоящим возрастом S3.
|
||||
Raw receipt: [clean-clone-final.json](evidence/clean-clone-final.json).
|
||||
Предыдущие неполные запуски сохранены и разобраны ниже; ошибочный ранний
|
||||
HTTP-порт отмечен INVALID_ENVIRONMENT и не считается проверкой API.
|
||||
Удалённый CI отдельно не подтверждён.
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
## Воспроизведение
|
||||
|
||||
## Архитектура и решения
|
||||
Нужны Git, Docker Desktop с Compose v2, uv0.13.7. Версии Python3.14.7,
|
||||
PostgreSQL18.3, Valkey8.1.3, HAProxy3.2.1, Pillow13.0.0 закреплены;
|
||||
MinIO/mc собираются из точных upstream commits в infra/s3.Dockerfile,
|
||||
Go module checksum проверяется GoSumDB. Сборка требует доступа к registries,
|
||||
Go module proxy и PyPI; наличие сети нельзя заменить уже установленной БД.
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```sh
|
||||
uv sync --locked
|
||||
make bootstrap
|
||||
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
make check validate PYTHON=.venv/bin/python
|
||||
make lint unit
|
||||
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 make test PYTHON=.venv/bin/python
|
||||
BASE_URL=http://127.0.0.1:8085 make test-student
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
URL: http://127.0.0.1:8085. API не имеет persistent volume/host port.
|
||||
Bootstrap создаёт только отсутствующие secrets и .env; не печатает значения,
|
||||
не перезаписывает существующие. `secrets/`0700, файлы0644 нужны UID10001
|
||||
через readonly Docker secrets mounts; сам каталог недоступен другим users.
|
||||
`docker compose down` сохраняет volumes. Удаление volumes — отдельное
|
||||
осознанное действие над своим синтетическим стендом.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
Для автоматических отказов включается только явный test override:
|
||||
|
||||
```sh
|
||||
export COMPOSE_FILE=compose.yaml:compose.test.yaml
|
||||
export NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1
|
||||
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-seed-repeat.json
|
||||
uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json --repeats 2
|
||||
uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-seed-repeat.json --output evidence/gc-repeat.json
|
||||
uv run --locked python -m scripts.verify_cluster --output evidence/cluster-repeat.json
|
||||
```
|
||||
|
||||
Проверки, меняющие состояние контейнеров, выполняются последовательно.
|
||||
Seed и результат — разные файлы; нельзя использовать seed с уже удалённым
|
||||
orphan. Verify ожидает настоящий возраст S3 не менее302s; часы/LastModified
|
||||
не подменяются. Для нового полного чистого повторения:
|
||||
|
||||
```sh
|
||||
# Перед стартом весь проверяемый source должен быть закоммичен и git clean.
|
||||
uv run --locked python scripts/verify_clean.py --output evidence/clean-clone-repeat.json
|
||||
```
|
||||
|
||||
Этот verifier создаёт отдельные secrets/env/volumes/image/project, собирает,
|
||||
запускает static/unit/public smoke/HTTP, cluster, всю fault matrix и cleanup,
|
||||
сверяет source hashes и OCI revision обоих API, worker, relay, reconciler и
|
||||
тестового controller; после удаляет только временный проект и его volumes.
|
||||
|
||||
## Границы устойчивости
|
||||
|
||||
1. API проверяет фактические bytes≤10MiB, MIME PNG/JPEG, декодирование,
|
||||
filename и ≤40M pixels до полной распаковки. Временный spool≤1MiB в RAM,
|
||||
затем bounded tmpfs; upload slots2/API, decode slot1/API, срок запроса20s.
|
||||
2. Устойчивый upload reservation120s COMMIT предшествует PUT в приватный
|
||||
`objects/staging/<UUID>/original`. После PUT одна PG transaction проверяет
|
||||
reservation с FOR UPDATE, создаёт Attachment queued + logical Job + Outbox,
|
||||
удаляет reservation и COMMIT. Только затем202 с захваченным queued-снимком.
|
||||
Быстрый worker не превращает upload-ответ в ready. Сам upload не имеет
|
||||
Idempotency-Key: потерянный HTTP-ответ нельзя безопасно повторять вслепую.
|
||||
3. API зависит от PG и S3 для ready, cache используется как необязательный
|
||||
read-through user cache. Broker не критичен для синхронного приёма:
|
||||
работа остаётся в outbox. Relay имеет PG+broker secrets, не S3/root keys.
|
||||
4. Relay читает due outbox FOR UPDATE SKIP LOCKED, XADD и WAITAOF1/0/2000
|
||||
**на одной выделенной connection**. Подтверждён fsync локального AOF;
|
||||
только затем published_at/COMMIT в PG. Crash в этом окне даёт дубль.
|
||||
5. Отдельный Valkey Streams broker с persistent volume, AOF appendfsync always,
|
||||
noeviction. Disposable cache — другой сервис/volume/policy. Нет Pub/Sub.
|
||||
6. Worker consumer group, COUNT=1 на каждый из2 слотов; XAUTOCLAIM возвращает
|
||||
зависшие PEL entries. PG атомарно переводит queued→processing, увеличивает
|
||||
attempt/fence, сохраняет owner/lease. Duplicate active оставляется pending;
|
||||
ready duplicate делает HEAD всех4вариантов/size/SHA metadata и ACK без resize.
|
||||
7. Child получает ID/fence, trusted source берётся из PG, не из произвольного
|
||||
URL или filename. До обработки проверяется исходный size/SHA. Все4 PUT идут
|
||||
в `objects/<attachment>/v<version>/<variant>`. Original побайтно неизменён.
|
||||
8. В одной PG transaction worker проверяет state/version/fence/live lease,
|
||||
публикует metadata всех вариантов, unique effect_publication и ready.
|
||||
Старый child может дописать свой version-prefix, но не подтвердить новый
|
||||
результат. Стабильный алгоритм нельзя менять под тем же pipeline_version.
|
||||
9. ACK только после устойчивой ready, failed/obsolete или решения queued retry.
|
||||
Watchdog relay восстанавливает expired leases; pending outbox сохраняет retry.
|
||||
Потерянные wakeups queued jobs переиздаются через15s без создания logical job.
|
||||
|
||||
## Состояния и варианты
|
||||
|
||||
GET показывает queued/processing/ready/failed. Queued/processing можно связать
|
||||
с Message; failed —409. Все metadata/content проходят Bearer+membership,
|
||||
чужой/неизвестный UUID —404, content до ready —409. У ready все3 производных
|
||||
варианта и error_code=null; size/content_type/SHA описывают original.
|
||||
Прямого redirect/public S3 URL нет. Atomic Message+Idempotency-Key сохранены.
|
||||
|
||||
EXIF orientation применяется; RGBA компонуется на белом. Thumbnail JPEG
|
||||
вписан≤256×256 без увеличения, floor/min1;320×200→256×160. Crop — центральный
|
||||
JPEG128×128 с допустимым увеличением маленького исходника. JPEGquality85,
|
||||
subsampling0,LANCZOS. Optimized — WebP≤1280×1280 без увеличения, quality82(v1)
|
||||
или76(v2), method4. EXIF/info удаляются из производных; metadata original
|
||||
сохраняются в исходных bytes. Маленький WebP не обязательно меньше original.
|
||||
|
||||
Job соответствует contracts/jobs.schema.json: schema_version1,
|
||||
type attachment.process.v1, UUID job/attachment, version, attempt>=1, RFC3339
|
||||
created_at; возможен W3C carrier. Bytes/credentials/paths/URL запрещены.
|
||||
Broker redelivery не меняет job_id; finite replay сохраняет job_id и увеличивает
|
||||
cycle; новая pipeline version получает новый job_id. Idempotency effect key —
|
||||
(attachment_id,pipeline_version,variant), PK в PG и детерминированный S3 key.
|
||||
|
||||
## Ограничение ресурсов и повторы
|
||||
|
||||
| Роль | CPU | RAM | Ограничение работы |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| API каждая из2 |0.5|768MiB|2uploads,1decode,128MiB tmpfs|
|
||||
| Worker |2|1536MiB|2child,COUNT1/slot,child RLIMIT_AS768MiB|
|
||||
| Relay |0.25|192MiB|один bounded publish/transaction|
|
||||
| Broker |0.5|256MiB|192MiB maxmemory,noeviction,AOF volume|
|
||||
| PostgreSQL |1|512MiB|pool10/API,4/worker-slot|
|
||||
| S3 |1|1024MiB|persistent volume,private bucket|
|
||||
| Reconciler |0.25|256MiB|inventory проход/60s|
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Normal timeout20s / lease23s; test override12s/15s даёт время контроллеру
|
||||
завершить v2 в другой slot пока v1 удерживается после PUT. Child действительно
|
||||
killable: parent убивает процесс по timeout, а не лишь отменяет future/thread.
|
||||
Весь container ограничен cgroup; лимит40M pixels не обещает, что всякий такой
|
||||
файл уложится в память и срок — memory_limit/timeout приводят к safe failed/retry.
|
||||
|
||||
## Эксперименты и отказы
|
||||
Максимум5attempts; временные ошибки/смерть worker/timeout → persisted next_at
|
||||
с jitter uniform0.25…min(30,2^attempt)s. Постоянные decode/checksum/memory ошибки
|
||||
сразу failed; временные после5→retry_exhausted. Публичные error_code безопасны,
|
||||
без stack/SQL/key. Реестр failed и audited replay доступны только оператору:
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
```sh
|
||||
docker compose exec worker python -m src.operator failed
|
||||
docker compose exec worker python -m src.operator replay --job-id UUID --operator maslov --reason 'Synthetic recovery demonstration'
|
||||
docker compose exec worker python -m src.operator version --job-id UUID --version 2 --operator maslov --reason 'Upgrade pipeline'
|
||||
docker compose run --rm --no-deps reconciler python -m src.reconcile
|
||||
docker compose run --rm --no-deps reconciler python -m src.reconcile --apply
|
||||
```
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Ready replay отвергается. Failed replay проверяет наличие source, оставляет
|
||||
audit и новый finite cycle. После source_expired нельзя обещать обработку
|
||||
отсутствующих bytes. New-version поддерживает реализованный переход1→2.
|
||||
|
||||
## Retention
|
||||
|
||||
Staging удаляется после безопасного COMMIT ready; сбой удаления не отменяет
|
||||
результат, последующий GC уберёт остаток. Failed/unconfirmed source и orphan
|
||||
живут минимум retention86400s, конфигурационный hard minimum300s. Cleanup
|
||||
по умолчанию dry-run; --apply явный. Active queued/processing source и prefix,
|
||||
действующий reservation и current published variants защищены. Перед delete
|
||||
повторяются PG lock/recheck и HEAD реального возраста. Expired reservation
|
||||
удаляется под lock; API не сможет поздно подтвердить удалённый source.
|
||||
Старые версии после успешной новой могут удаляться; metadata/audit сохраняются.
|
||||
Failed без source остаётся failed/source_expired и replay отвергается.
|
||||
Текущий original не удаляется лишь из-за отсутствия Message links.
|
||||
|
||||
## Исследование: гипотеза и инварианты
|
||||
|
||||
Гипотеза: durable intent/outbox + PG version/fence/lease CAS + immutable
|
||||
version keys сохраняют подтверждённую работу и не допускают publication старой
|
||||
версии поверх новой при at-least-once delivery.
|
||||
|
||||
Проверяемые инварианты: каждый202 имеет durable Job; все принятые matrix jobs
|
||||
доходят до ready или наблюдаемого failed; финальный ready имеет ровно4 variant
|
||||
rows/keys и1effect_publication на версию; original SHA равен загрузке;
|
||||
old version не меняет hashes текущей; конечный retry/replay наблюдаемы;
|
||||
cleanup не удаляет current/active/fresh. До handoff crash не получает202.
|
||||
|
||||
`scripts/verify_pipeline.py --repeats2` использует отдельный test controller
|
||||
с PG/S3/broker secrets (API не получает broker secrets), single-use hooks в PG
|
||||
с durable hit_at, реальные container SIGKILL и повторный запуск. Hooks по
|
||||
умолчанию выключены, публичного endpoint для них нет. Проверяются:
|
||||
source-before-handoff, COMMIT-before-publish, confirmed broker SIGKILL/volume,
|
||||
relay XADD-before-PG-marker, worker before_effect/after_effect/after_commit
|
||||
каждый дважды; transient, timeout, permanent/replay,5attempt exhaustion/replay,
|
||||
duplicates/reorder, held stalev1 послеv2 и старые notifications; malformed
|
||||
notification, private IAM/anonymous denial и другая API после uploader stop.
|
||||
|
||||
Raw evidence сохраняет IDs, attempts, fences, events,4keys/checksums и
|
||||
publications. Worker child_finished даёт elapsed/CPU/peakRSS; это длительность
|
||||
child, не полный queue lag. Исследование про корректность, не максимальный RPS.
|
||||
|
||||
## Вопросы защиты
|
||||
|
||||
**Publisher confirm и consumer ACK?** Первый подтверждает durable приём брокером
|
||||
(здесь локальный AOF fsync через WAITAOF), второй — устойчивое завершение или
|
||||
сохранённое решение о повторе у consumer. Между PG/Valkey/S3 нет общей transaction.
|
||||
|
||||
**Что уже сохранено к202?** Полный проверенный original в private persistent
|
||||
S3 staging плюс attachment/job/outbox в одной подтверждённой PG transaction.
|
||||
Смерть API/worker не удаляет намерение. Перед202 тяжёлого resize нет.
|
||||
|
||||
**Почему дубли нормальны?** Producer/consumer могут потерять ответ после
|
||||
успешного действия. Отправитель повторяет, поскольку не знает исход. Logical
|
||||
job_id и effect tuple/version/fence позволяют повторить доставку безопасно.
|
||||
Ровно одна publication в PG не означает exactly-once delivery в broker.
|
||||
|
||||
**Как отличить зависший worker от долгого?** Совершенно точно нельзя: lease —
|
||||
выбранный бюджет. Parent timeout убивает child; watchdog восстанавливает работу
|
||||
после lease, новый fence делает старое завершение недействительным. Ограничение
|
||||
slots/COUNT сдерживает конкуренцию. Возможен лишний PUT, но не старый COMMIT.
|
||||
|
||||
## Источники, помощь, ограничения
|
||||
|
||||
Использованы текущие teacher contracts, собственная ЛР4, pinned dependencies,
|
||||
[Valkey WAITAOF](https://valkey.io/commands/waitaof/),
|
||||
[Valkey XAUTOCLAIM](https://valkey.io/commands/xautoclaim/),
|
||||
[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html).
|
||||
При проектировании, реализации, тестах и отчёте использована помощь ИИ.
|
||||
Код/данные/история других студентов не использовались; результаты — реальные
|
||||
локальные запуски, а remote CI отдельно не считается прошедшим.
|
||||
|
||||
Один host/LB/PG/S3/broker — SPOF, AOF/fsync/volume не заменяет backup и защиту
|
||||
от потери диска. HTTP допустим для локальных синтетических данных; удалённый
|
||||
доступ требует TLS. Строгого production exactly-once нет. HEAD metadata не
|
||||
заменяет полный periodic checksum audit. Удалённые bytes не восстановить по SHA.
|
||||
Весь inventory GC хранится в RAM, outbox/events/audit требуют retention при
|
||||
долгой эксплуатации. Нет global admission quota: при долгой недоступности
|
||||
worker/broker PG outbox может расти; исследование перегрузки — другой трек.
|
||||
Short synthetic fixtures и повторы ограничивают обобщение на большие файлы.
|
||||
Успешно завершившиеся child samples не включают RSS убитых процессов.
|
||||
|
||||
## Настройки
|
||||
|
||||
Compose использует PUBLIC_PORT, API_REPLICAS/CPUS/MEMORY, DB_POOL_SIZE,
|
||||
SESSION_TTL_SECONDS, WORKER_CONCURRENCY, JOB_MAX_ATTEMPTS/TIMEOUT/LEASE и
|
||||
STAGING_RETENTION_SECONDS. Значения .env.example для S3 endpoint/region/bucket,
|
||||
MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и idempotency TTL документируют фиксированный
|
||||
контракт реализации: s3:9000/us-east-1/messenger,10MiB/40M/24h. Их нельзя изменить
|
||||
одним редактированием env: соответствующие constants/SQL надо менять согласованно.
|
||||
Secret *_FILE paths заданы в compose; значения создаёт bootstrap.
|
||||
|
||||
## Карта критериев и история проверок
|
||||
|
||||
| Уровень / требование | Реализация / автоматическая проверка |
|
||||
| --- | --- |
|
||||
| 3: минимум ЛР4 +202queued | API/ACL/byte/pixel/MIME guards;112 HTTP tests, public smoke |
|
||||
| 3: durable broker и отдельный worker | persistent Valkey Streams/AOF/WAITAOF; broker SIGKILL с volume |
|
||||
| 3:4states, original/thumbnail, retry | src/worker.py, schema-valid metadata; failed/retry scenarios |
|
||||
| 3:worker stop→queued link→restart→ready | commit_before_publish_and_worker_stopped |
|
||||
| 4:crop/optimized/metadata removal | HTTP geometry/EXIF/alpha +22 unit tests |
|
||||
| 4:outbox gap и publisher crash | API COMMIT-before-publish и relay XADD-before-PG-marker |
|
||||
| 4:finite backoff/permanent/transient/replay |5attempt exhaustion, timeout, failed registry, audit, ready replay rejection |
|
||||
| 4:crash после PUT доACK, lease, bounded slots | before_effect/after_effect/after_commit ×2; live state/actual limits подтверждены clean-clone-final |
|
||||
| 4:cleanup после фиксации/retention | verify_cleanup seed/verify, реальныйS3возраст, dry/apply/repeat |
|
||||
| 5:versioned idempotency track | duplicates/reorder и held stalev1→v2×2, no old publication/hash change |
|
||||
|
||||
[evidence/http-first.json](evidence/http-first.json):112 HTTP PASS404.42s,
|
||||
source/runtime bb4e8a2. Отдельные22unit PASS после добавления real pixel,
|
||||
checksum и malformed image worker tests. Public smoke PASS.
|
||||
|
||||
Первые полные matrix runs сохранены честно как незавершённые:
|
||||
[pipeline-first-failed.json](evidence/pipeline-first-failed.json) —2cases,
|
||||
затем тест сравнил очередь при работающем relay; snapshot изолирован остановкой
|
||||
relay, повтор подтвердил сохранность exact entries после broker SIGKILL.
|
||||
[pipeline-second-failed.json](evidence/pipeline-second-failed.json) —15cases
|
||||
прошли, затем отдельные controller invocations съели12s бюджет старого child.
|
||||
V2 была ready, но parent успел убитьv1; это ещё не доказательство CAS.
|
||||
|
||||
Новая координация использует один заранее запущенный controller: ждёт phase,
|
||||
создаётv2, ждёт ready, отпускаетv1 и видит stale_completion_rejected внутри
|
||||
живого окна. [pipeline-versions-first.json](evidence/pipeline-versions-first.json)
|
||||
— **scope=versions**,4cases PASS:2version races, malformed notification,
|
||||
private/IAM/otherAPI. Это целевой повтор, а не19cases полного matrix.
|
||||
Приложение в этих исправлениях harness не менялось. Финальный clean clone
|
||||
завершил scope=all, включая строгие ACK/live-state/resource assertions.
|
||||
|
||||
[evidence/gc-first.json](evidence/gc-first.json): реальный возраст orphan
|
||||
1279.103s (выше300s); dry-run сохранил bytes, apply удалил expired objects,
|
||||
повтор не изменил ничего. Active/current/fresh и неистёкший reservation
|
||||
защищены; source_expired replay отвергнут; expired reservation больше не
|
||||
проходит handoff predicate. Fixture metadata failed состарены SQL отдельно,
|
||||
но возраст S3 LastModified не подменялся. Cleanup fixtures синтетические и
|
||||
не обещают восстановление bytes из checksum.
|
||||
|
||||
Для финальной проверки worker перед fault hook сохраняет diagnostic event
|
||||
worker_resource_limits с фактическим RLIMIT_AS и cgroup memory.max/cpu.max.
|
||||
Сбор показаний самим child исключает влияние медленного внешнего DockerCLI
|
||||
на12s test budget. Это диагностическое добавление также проверяется новым
|
||||
образом из чистого clone; предыдущие HTTP/source evidence не переименованы.
|
||||
|
||||
Первый cluster receipt [cluster-first-failed.json](evidence/cluster-first-failed.json)
|
||||
сохраняет успешные отказы двухAPI. Rolling занял дольше45s окна, поэтому
|
||||
генератор завершился сразу с action и не получил post-action observation.
|
||||
Verifier теперь обязательно наблюдает ещё2s после action, сохраняя все
|
||||
проверки данных и failover≤10s. Это исправление наблюдения, не ослабление SLA.
|
||||
|
||||
[evidence/cluster-first.json](evidence/cluster-first.json): complete PASS;
|
||||
API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762.
|
||||
Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены.
|
||||
Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились,
|
||||
прежний token и file сохранились. Это не zero-error rolling.
|
||||
|
||||
## Уточнение строгого окна COMMIT→ACK
|
||||
|
||||
Первый чистый повтор [clean-clone-first.json](evidence/clean-clone-first.json)
|
||||
sourceeb6d229 прошёл112HTTP379.34s/22unit/19cases/cluster/cleanup/hashes.
|
||||
Однако в after_commit snapshot уже был duplicate_decision: второйconsumer
|
||||
XAUTOCLAIM после3s успевал ACK раньшеSIGKILL. Это безопасно для данных,
|
||||
но не доказывает конкретное окно COMMIT-before-ACK. Первый receipt сохранён
|
||||
как результат прежней версии проверок; строгая граница требует нового повтора.
|
||||
|
||||
Reclaim idle теперь равен lease(23s normal/15s test), превышая child timeout
|
||||
20s/12s. Это предотвращает ранний захват чужого live entry. Новый тест
|
||||
снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и
|
||||
отсутствие child_finished; после restart требует ready duplicate decision
|
||||
и исчезновение notifications. Scope=all, данные и все прежние assertions
|
||||
сохранены. Финальное доказательство — clean-clone-final.json (PASS).
|
||||
|
||||
## Финальный результат из чистого clone
|
||||
|
||||
Дата запуска UTC: 2026-11-15T20:48:55.825623+00:00. Source: `442163b0f0a549ef065cb13ee39b5ca2f8580c18`.
|
||||
Последующие изменения отчёта/evidence не меняют исполняемые исходники.
|
||||
Host: Apple M1, 8CPU, 8GiB RAM; Docker VM около6GiB RAM. Лимиты
|
||||
контейнеров перечислены выше и сохранены в cluster runtime snapshots.
|
||||
Новые secrets/env/volumes, отдельный image/project/port; после проверки
|
||||
удалён только временный проект. Основные данные сохранены.
|
||||
|
||||
HTTP: `112 passed in 397.50s (0:06:37)`. Unit:22PASS; public smoke,
|
||||
check/validate/lint, git fsck и сверка SHA каждого src/*.py с OCI revision
|
||||
всех6контейнеров (2API/worker/relay/reconciler/controller) — PASS.
|
||||
|
||||
| Сценарий | Итоговые jobs: version/state/attempt/cycle/fence | Publications | Rows/keys |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| staging_before_handoff_crash | см. raw assertions | — | — |
|
||||
| commit_before_publish_and_worker_stopped | v1/ready/1/1/1 | 1 | 4/4 |
|
||||
| confirmed_broker_entry_survives_sigkill | v1/ready/1/1/1 | 1 | 4/4 |
|
||||
| relay_crash_after_xadd_before_pg_marker | v1/ready/1/1/1 | 1 | 4/4 |
|
||||
| before_effect_sigkill_repeat_1 | v1/ready/2/1/3 | 1 | 4/4 |
|
||||
| after_effect_sigkill_repeat_1 | v1/ready/2/1/3 | 1 | 4/4 |
|
||||
| after_commit_sigkill_repeat_1 | v1/ready/1/1/1 | 1 | 4/4 |
|
||||
| before_effect_sigkill_repeat_2 | v1/ready/2/1/3 | 1 | 4/4 |
|
||||
| after_effect_sigkill_repeat_2 | v1/ready/2/1/3 | 1 | 4/4 |
|
||||
| after_commit_sigkill_repeat_2 | v1/ready/1/1/1 | 1 | 4/4 |
|
||||
| transient_retry_with_jitter | v1/ready/2/1/3 | 1 | 4/4 |
|
||||
| hung_child_killed_and_retried | v1/ready/2/1/3 | 1 | 4/4 |
|
||||
| permanent_failed_registry_and_audited_replay | v1/ready/1/2/4 | 1 | 4/4 |
|
||||
| finite_retry_exhaustion_then_explicit_replay | v1/ready/1/2/12 | 1 | 4/4 |
|
||||
| reordered_and_duplicate_notifications | v1/ready/1/1/1; v1/ready/1/1/1 | 1; 1 | 4/4; 4/4 |
|
||||
| stale_v1_after_v2_repeat_1 | v1/obsolete/1/1/2; v2/ready/1/1/1 | 0; 1 | 0/4; 4/4 |
|
||||
| stale_v1_after_v2_repeat_2 | v1/obsolete/1/1/2; v2/ready/1/1/1 | 0; 1 | 0/4; 4/4 |
|
||||
| malformed_notification_rejected | см. raw assertions | — | — |
|
||||
| private_iam_and_other_api_download | см. raw assertions | — | — |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Полные UUID/attempts/fences/events/checksums — в
|
||||
[pipeline-final.json](evidence/pipeline-final.json) и
|
||||
[effects.csv](evidence/pipeline-analysis-final/effects.csv).
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Строгие COMMIT→ACK snapshots перед SIGKILL:
|
||||
|
||||
- `after_commit_sigkill_repeat_1`: job `efd4aa67-b47e-462f-8e86-614c5dbb8a17`, pending entries `1794776649599-0`; ready/publication1, child_finished отсутствует.
|
||||
- `after_commit_sigkill_repeat_2`: job `c422c6de-4c05-43ea-b748-e4ff856a70c8`, pending entries `1794776766791-0`; ready/publication1, child_finished отсутствует.
|
||||
|
||||
После restart оба сценария дождались duplicate_decision(state=ready) и
|
||||
отсутствия job notifications в stream (ACK+XDEL), без нового resize.
|
||||
Два held stalev1 действительно дошли до stale_completion_rejected после
|
||||
readyv2; уv1 ноль publications, hashes всех4currentv2 не изменились.
|
||||
|
||||
Агрегация: 17 ready observations — каждое с1publication,4rows/4keys;
|
||||
2 obsolete observations — с0publications. Это конечные наблюдения
|
||||
синтетической матрицы, а не формальное доказательство для всех возможных расписаний.
|
||||
|
||||

|
||||
|
||||
15 successful child samples: callback elapsed 0.2563–0.4592s,
|
||||
CPU 0.2274–0.3770s, peakRSS 79.12–79.59MiB.
|
||||
Elapsed/CPU измеряют тело callback и не включают spawn/import до него; RSS
|
||||
учитывает startup процесса. Убитые child не входят в successful samples.
|
||||
Это не queue lag, p95 до ready или throughput/SLA. Фактические лимиты
|
||||
наблюдал сам child: RLIMIT_AS soft/hard805306368bytes, cgroup memory.max
|
||||
1610612736bytes; CPU quota — в observed_limits raw case.
|
||||
|
||||
Cluster failovers (confirmed writes / transient errors / recovery seconds):
|
||||
|
||||
- 202 / 10 / 4.776s; confirmed data preserved, duplicate texts отсутствуют.
|
||||
- 218 / 8 / 2.971s; confirmed data preserved, duplicate texts отсутствуют.
|
||||
|
||||
Rolling: 739 confirmed writes, 45 transient errors;
|
||||
исходные API CID заменены, replay и данные сохранены. Public failover и
|
||||
persistence с прежними token/file прошли; StartedAt хранилищ изменился.
|
||||
Кратковременные ошибки учитываются, zero-error rolling не заявлен.
|
||||
|
||||
Cleanup: реальный возраст S3 orphan 754.765s;
|
||||
dry-run 10actions без изменения bytes; apply 10actions;
|
||||
repeat0actions. Active/current/fresh и действующий reservation защищены,
|
||||
expired source replay отвергнут, expired writer fenced. Возраст fixture
|
||||
metadata менялся SQL отдельно; S3 LastModified не подменялся.
|
||||
Raw: [cleanup-final.json](evidence/cleanup-final.json).
|
||||
|
||||
## Публикация
|
||||
|
||||
Публичный fork: [wh1tet3a/mtusi2026lab5](https://gitea.privcloud.tech/wh1tet3a/mtusi2026lab5).
|
||||
PR: [ЛР 5 — БВТ2301 — Маслов — на 5, №70](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/70),
|
||||
ветка `solution/bvt2301-maslov` → преподавательская `main`.
|
||||
При публикации remote CI имел pending/0checks; локальный clean PASS не выдаётся
|
||||
за результат удалённого CI. Commit со ссылкой на PR меняет только документацию.
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# PostgreSQL / S3 / broker — ЛР5
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[Client] --> LB[HAProxy]
|
||||
LB --> A[API 1/2]
|
||||
A --> S[Private S3 staging]
|
||||
A --> P[(PG attachment + job + outbox)]
|
||||
P --> R[Relay + watchdog]
|
||||
R --> B[(Valkey Streams + AOF)]
|
||||
B --> W[Worker 2 bounded child slots]
|
||||
W --> V[S3 versioned original + 3 variants]
|
||||
W --> E[(PG fenced atomic publication)]
|
||||
E --> A
|
||||
```
|
||||
|
||||
Core: users→sessions; chats↔members; chats→messages; message_attachments
|
||||
имеет составные FK chat/message и chat/attachment. Сессии и idempotency
|
||||
сохраняются в PG; cache не является источником истины. Row locks и unique
|
||||
(user,method,path,key) сохраняют Message+result атомарно.
|
||||
|
||||
Attachment хранит свойства original, target pipeline_version и
|
||||
published_version. Jobs unique(attachment_id,pipeline_version), UUID logical ID,
|
||||
state,attempt/cycle,fence,lease/owner,next_at,safeerror. Outbox содержит только
|
||||
schema-valid envelope, due/published. Consumer group хранит pending entries,
|
||||
но окончательные решения находятся в PG. Failed registry — jobs.state=failed.
|
||||
|
||||
Результаты: attachment_variants PK(attachment_id,pipeline_version,name),
|
||||
unique object_key; effect_publications PK(attachment_id,pipeline_version).
|
||||
Worker в одной транзакции проверяет target version, fence, live lease и
|
||||
processing, записывает4variants+effect_publication+ready. Object bytes
|
||||
предшествуют COMMIT; дубли могут повторить PUT тех же bytes/keys.
|
||||
Публикацию новой версии нельзя перезаписать старым fence/version.
|
||||
|
||||
Upload reservation120s предотвращает поздний COMMIT после GC: row lock плюс
|
||||
проверка expiry нужны и при handoff, и при удалении просроченного reservation.
|
||||
Unconfirmed bytes не становятся accepted jobs сами по себе.
|
||||
|
||||
S3: staging `objects/staging/<UUID>/original`, final
|
||||
`objects/<attachment>/v<version>/{original,thumbnail,crop,optimized}`.
|
||||
Filename — только отображаемое поле, не путь. IAM ограничен messenger/objects/*,
|
||||
корневые credentials получает только S3/bootstrap. PG содержит metadata/SHA,
|
||||
а не bytes. Broker envelope содержит ID/version/attempt/time, не storage paths.
|
||||
|
||||
pipeline_audit сохраняет explicit operator replay/version; pipeline_events —
|
||||
наблюдаемые решения без secrets. pipeline_faults — test-only single-use hooks,
|
||||
выключены без PIPELINE_FAULTS=1. В production API не имеет broker credentials;
|
||||
test controller появляется только в compose.test.yaml и не публикует портов.
|
||||
@@ -0,0 +1,30 @@
|
||||
# Explicit test-only fault instrumentation, never enabled by default.
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
PIPELINE_FAULTS: "1"
|
||||
relay:
|
||||
environment:
|
||||
PIPELINE_FAULTS: "1"
|
||||
worker:
|
||||
environment:
|
||||
PIPELINE_FAULTS: "1"
|
||||
JOB_TIMEOUT_SECONDS: "12"
|
||||
JOB_LEASE_SECONDS: "15"
|
||||
controller:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
pull_policy: never
|
||||
command: [python, -c, "import time; time.sleep(86400)"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
|
||||
cpus: 0.5
|
||||
mem_limit: 192m
|
||||
memswap_limit: 192m
|
||||
read_only: true
|
||||
healthcheck: {disable: true}
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
name: mtusi-lab5-maslov
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
memswap_limit: 512m
|
||||
cache:
|
||||
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
command: [valkey-server, /run/secrets/valkey.conf]
|
||||
secrets:
|
||||
- cache_password
|
||||
- source: valkey_config
|
||||
target: valkey.conf
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
pull_policy: never
|
||||
command: [python, -m, src.migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
read_only: true
|
||||
api:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-unknown}
|
||||
scale: ${API_REPLICAS:-2}
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
|
||||
secrets: [database_url, cache_url, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
cache: {condition: service_healthy}
|
||||
s3-bootstrap: {condition: service_completed_successfully}
|
||||
cpus: ${API_CPUS:-0.5}
|
||||
mem_limit: ${API_MEMORY:-384m}
|
||||
memswap_limit: ${API_MEMORY:-384m}
|
||||
pids_limit: 128
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=128m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
stop_grace_period: 10s
|
||||
s3:
|
||||
image: ${S3_IMAGE:-mtusi-storage:minio-20251015}
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
cpus: 1
|
||||
mem_limit: 1024m
|
||||
memswap_limit: 1024m
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,nosuid,size=32m"]
|
||||
environment:
|
||||
HOME: /tmp
|
||||
security_opt: [no-new-privileges:true]
|
||||
s3-bootstrap:
|
||||
image: ${MC_IMAGE:-mtusi-storage:mc-20250813}
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: bootstrap}
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,nosuid,size=16m"]
|
||||
reconciler:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
pull_policy: never
|
||||
command: [python, -m, src.reconcile, --apply, --loop]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
secrets: [database_url, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-bootstrap: {condition: service_completed_successfully}
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
memswap_limit: 256m
|
||||
read_only: true
|
||||
healthcheck: {disable: true}
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
command: [valkey-server, /run/secrets/broker.conf]
|
||||
secrets:
|
||||
- broker_password
|
||||
- source: broker_config
|
||||
target: broker.conf
|
||||
volumes: [broker_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
memswap_limit: 256m
|
||||
relay:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
pull_policy: never
|
||||
command: [python, -m, src.pipeline]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
secrets: [database_url, broker_url]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
memswap_limit: 192m
|
||||
read_only: true
|
||||
healthcheck: {disable: true}
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
pull_policy: never
|
||||
command: [python, -m, src.worker]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23}
|
||||
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
s3-bootstrap: {condition: service_completed_successfully}
|
||||
cpus: 2
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
pids_limit: 128
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
|
||||
healthcheck: {disable: true}
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
stop_grace_period: 25s
|
||||
lb:
|
||||
image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9
|
||||
ports:
|
||||
- "127.0.0.1:${PUBLIC_PORT:-8085}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,nosuid,size=4m"]
|
||||
security_opt: [no-new-privileges:true]
|
||||
stop_grace_period: 10s
|
||||
secrets:
|
||||
broker_password: {file: ./secrets/broker_password}
|
||||
broker_url: {file: ./secrets/broker_url}
|
||||
broker_config: {file: ./secrets/broker.conf}
|
||||
s3_root_user: {file: ./secrets/s3_root_user}
|
||||
s3_root_password: {file: ./secrets/s3_root_password}
|
||||
s3_access_key: {file: ./secrets/s3_access_key}
|
||||
s3_secret_key: {file: ./secrets/s3_secret_key}
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_password:
|
||||
file: ./secrets/cache_password
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
valkey_config:
|
||||
file: ./secrets/valkey.conf
|
||||
name: valkey.conf
|
||||
volumes:
|
||||
broker_data:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18",
|
||||
"seed": "clean-gc-seed.json",
|
||||
"complete": true,
|
||||
"real_s3_age_seconds": 754.765394,
|
||||
"dry_run": {
|
||||
"apply": false,
|
||||
"retention_seconds": 300,
|
||||
"actions": [
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original",
|
||||
"applied": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"apply": {
|
||||
"apply": true,
|
||||
"retention_seconds": 300,
|
||||
"actions": [
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original",
|
||||
"applied": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"repeat": {
|
||||
"apply": true,
|
||||
"retention_seconds": 300,
|
||||
"actions": []
|
||||
},
|
||||
"before": {
|
||||
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": true,
|
||||
"objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": true,
|
||||
"objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": true,
|
||||
"objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true,
|
||||
"objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true,
|
||||
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true
|
||||
},
|
||||
"after": {
|
||||
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": false,
|
||||
"objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": false,
|
||||
"objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": false,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": false,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": false,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": false,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": false,
|
||||
"objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true,
|
||||
"objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true,
|
||||
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true,
|
||||
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true
|
||||
},
|
||||
"dry_run_unchanged": true,
|
||||
"active_current_and_fresh_protected": true,
|
||||
"expired_source_replay_refused": true,
|
||||
"expired_writer_fenced": true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,20 @@
|
||||
Traceback (most recent call last):
|
||||
File "<frozen runpy>", line 203, in _run_module_as_main
|
||||
File "<frozen runpy>", line 88, in _run_code
|
||||
File "<repository>/scripts/verify_cluster.py", line 342, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "<repository>/scripts/verify_cluster.py", line 278, in main
|
||||
report["rolling"] = exercise(
|
||||
~~~~~~~~^
|
||||
client,
|
||||
^^^^^^^
|
||||
...<3 lines>...
|
||||
seconds=45,
|
||||
^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "<repository>/scripts/verify_cluster.py", line 114, in exercise
|
||||
assert recovery is not None and (seconds > 20 or recovery <= 10)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
AssertionError
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,209 @@
|
||||
{
|
||||
"source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88",
|
||||
"seed": "gc-seed-first.json",
|
||||
"complete": true,
|
||||
"real_s3_age_seconds": 1279.102705,
|
||||
"dry_run": {
|
||||
"apply": false,
|
||||
"retention_seconds": 300,
|
||||
"actions": [
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original",
|
||||
"applied": false
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original",
|
||||
"applied": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"apply": {
|
||||
"apply": true,
|
||||
"retention_seconds": 300,
|
||||
"actions": [
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original",
|
||||
"applied": true
|
||||
},
|
||||
{
|
||||
"action": "delete_expired_object",
|
||||
"key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original",
|
||||
"applied": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"repeat": {
|
||||
"apply": true,
|
||||
"retention_seconds": 300,
|
||||
"actions": []
|
||||
},
|
||||
"before": {
|
||||
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": true,
|
||||
"objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": true,
|
||||
"objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": true,
|
||||
"objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true,
|
||||
"objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true,
|
||||
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true
|
||||
},
|
||||
"after": {
|
||||
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": false,
|
||||
"objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": false,
|
||||
"objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": false,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": false,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": false,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": false,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": false,
|
||||
"objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true,
|
||||
"objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true,
|
||||
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true,
|
||||
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true
|
||||
},
|
||||
"dry_run_unchanged": true,
|
||||
"active_current_and_fresh_protected": true,
|
||||
"expired_source_replay_refused": true,
|
||||
"expired_writer_fenced": true
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"seed_id": "ae4b883f7b784f40ae8f5e1c78ac3091",
|
||||
"source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2",
|
||||
"date": "2026-11-15T19:46:23.318659+00:00",
|
||||
"ready_attachment": "d6f10e70-74e9-4394-be76-5c0b14d3f91f",
|
||||
"ready_job": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35",
|
||||
"chat": "3428fad7-2178-4730-b56a-4b579bdd4966",
|
||||
"owner": "36007eaa-ceba-4f80-98fa-13c912f73873",
|
||||
"version2_job": "cc18c75e-8886-4333-bae8-5da93d626c73",
|
||||
"orphan": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan",
|
||||
"failed_id": "b98bc768-f36a-4615-9097-c93a78bfed05",
|
||||
"active_id": "69150761-a8a1-437b-91b0-b363d7a1507a",
|
||||
"reservation_id": "ec7a6893-8d3b-4488-b0cd-06e53499cf9d",
|
||||
"expired_reservation_id": "402b3515-91ad-40e0-8e36-6627feb18be7",
|
||||
"failed_source": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original",
|
||||
"active_source": "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original",
|
||||
"reservation_key": "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original",
|
||||
"expired_reservation_key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original",
|
||||
"seeded_unix": 1794771995.922627
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"source_commit": "bb4e8a2",
|
||||
"runtime_revision": "bb4e8a2",
|
||||
"public_smoke": "PASS",
|
||||
"http_tests": 112,
|
||||
"passed": 112,
|
||||
"failed": 0,
|
||||
"seconds": 404.42,
|
||||
"unit_tests": 19,
|
||||
"unit_result": "PASS",
|
||||
"output": "........................................................................ [ 64%]\n........................................ [100%]\n112 passed in 404.42s (0:06:44)\n"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"result":"INVALID_ENVIRONMENT","cause":"Inherited student fixture default pointed to stopped lab4 port8084, not lab5 port8085; all connections refused; no API correctness conclusion.","failed":40,"errors":72,"seconds":8.77}
|
||||
@@ -0,0 +1,20 @@
|
||||
case,role,job_id,version,attempt,cycle,fence,state,publications,variant_rows,physical_keys
|
||||
commit_before_publish_and_worker_stopped,effect,3e2eccd9-713d-4982-a025-5537b66c114c,1,1,1,1,ready,1,4,4
|
||||
confirmed_broker_entry_survives_sigkill,effect,5747edc1-9de6-40ba-a143-85ded56744d1,1,1,1,1,ready,1,4,4
|
||||
relay_crash_after_xadd_before_pg_marker,effect,2177d978-9a49-4856-a9e9-6211e94e9246,1,1,1,1,ready,1,4,4
|
||||
before_effect_sigkill_repeat_1,effect,3f046f7c-825f-446f-bf52-07e4cd7e330b,1,2,1,3,ready,1,4,4
|
||||
after_effect_sigkill_repeat_1,effect,64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0,1,2,1,3,ready,1,4,4
|
||||
after_commit_sigkill_repeat_1,effect,efd4aa67-b47e-462f-8e86-614c5dbb8a17,1,1,1,1,ready,1,4,4
|
||||
before_effect_sigkill_repeat_2,effect,6eacee3c-6b6b-4435-bd05-bb50c29df24f,1,2,1,3,ready,1,4,4
|
||||
after_effect_sigkill_repeat_2,effect,0d40fa0a-e6b9-49cb-907d-e82faa346906,1,2,1,3,ready,1,4,4
|
||||
after_commit_sigkill_repeat_2,effect,c422c6de-4c05-43ea-b748-e4ff856a70c8,1,1,1,1,ready,1,4,4
|
||||
transient_retry_with_jitter,effect,00893137-7e67-434b-8764-5c85ed5353d2,1,2,1,3,ready,1,4,4
|
||||
hung_child_killed_and_retried,effect,eefeceb7-6e35-4358-bbac-772170b964fa,1,2,1,3,ready,1,4,4
|
||||
permanent_failed_registry_and_audited_replay,effect,88929609-e34a-4b5f-ad36-ffce561ebde8,1,1,2,4,ready,1,4,4
|
||||
finite_retry_exhaustion_then_explicit_replay,effect,b6e323c3-3e4a-45fd-a4a5-709094e97485,1,1,2,12,ready,1,4,4
|
||||
reordered_and_duplicate_notifications,1,e7f4c924-cb49-40a5-b15b-b5bd8b3bd401,1,1,1,1,ready,1,4,4
|
||||
reordered_and_duplicate_notifications,2,108ade45-7571-4362-92d5-bde3c6232fe9,1,1,1,1,ready,1,4,4
|
||||
stale_v1_after_v2_repeat_1,v1,bc562c08-8221-4f9d-b5ff-890d16987495,1,1,1,2,obsolete,0,0,4
|
||||
stale_v1_after_v2_repeat_1,v2,75ceffdc-919b-4f92-96c4-2e0c775850d7,2,1,1,1,ready,1,4,4
|
||||
stale_v1_after_v2_repeat_2,v1,61e16816-661d-46bd-882e-80cc1cab86dd,1,1,1,2,obsolete,0,0,4
|
||||
stale_v1_after_v2_repeat_2,v2,6d569c1e-c44c-4b65-a49d-ae731029270a,2,1,1,1,ready,1,4,4
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18",
|
||||
"cases": 19,
|
||||
"observed_ready_effects": 17,
|
||||
"observed_obsolete_effects": 2,
|
||||
"all_ready_have_one_publication_and_four_variants": true,
|
||||
"all_obsolete_have_zero_publications": true,
|
||||
"successful_child_samples": 15,
|
||||
"child_elapsed_range_seconds": [
|
||||
0.2563110829996731,
|
||||
0.4591874589996223
|
||||
],
|
||||
"child_cpu_range_seconds": [
|
||||
0.22742345900000016,
|
||||
0.37699009000000006
|
||||
],
|
||||
"child_peak_rss_range_mib": [
|
||||
79.125,
|
||||
79.5859375
|
||||
],
|
||||
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2",
|
||||
"date": "2026-11-15T19:46:36.154413+00:00",
|
||||
"track": 2,
|
||||
"complete": false,
|
||||
"cases": [
|
||||
{
|
||||
"case": "staging_before_handoff_crash",
|
||||
"no_202": true,
|
||||
"no_attachment_or_job_created": true,
|
||||
"orphan_retained_for_cleanup": true
|
||||
},
|
||||
{
|
||||
"case": "commit_before_publish_and_worker_stopped",
|
||||
"attachment": "f554e320-4881-4087-b813-9f28607fc3b1",
|
||||
"effect": {
|
||||
"job": {
|
||||
"id": "1016e737-7ec1-4fde-9946-a342d70b2a71",
|
||||
"attachment_id": "f554e320-4881-4087-b813-9f28607fc3b1",
|
||||
"pipeline_version": 1,
|
||||
"source_key": "objects/staging/f554e320-4881-4087-b813-9f28607fc3b1/original",
|
||||
"state": "ready",
|
||||
"attempt": 1,
|
||||
"cycle": 1,
|
||||
"fence": 1,
|
||||
"lease_owner": null,
|
||||
"lease_until": null,
|
||||
"next_at": "2026-11-15 19:47:10.665115+00:00",
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-15 19:47:10.665115+00:00",
|
||||
"updated_at": "2026-11-15 19:47:36.689673+00:00"
|
||||
},
|
||||
"variants": [
|
||||
{
|
||||
"name": "crop",
|
||||
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop",
|
||||
"sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef",
|
||||
"size_bytes": 1077
|
||||
},
|
||||
{
|
||||
"name": "optimized",
|
||||
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized",
|
||||
"sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8",
|
||||
"size_bytes": 194
|
||||
},
|
||||
{
|
||||
"name": "original",
|
||||
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original",
|
||||
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df",
|
||||
"size_bytes": 742
|
||||
},
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail",
|
||||
"sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199",
|
||||
"size_bytes": 1749
|
||||
}
|
||||
],
|
||||
"publications": 1,
|
||||
"physical_keys": [
|
||||
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop",
|
||||
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized",
|
||||
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original",
|
||||
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"event": "claimed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"attempt": 1,
|
||||
"version": 1
|
||||
},
|
||||
"created_at": "2026-11-15 19:47:34.529681+00:00"
|
||||
},
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794772054509-0",
|
||||
"outbox": 11
|
||||
},
|
||||
"created_at": "2026-11-15 19:47:34.584052+00:00"
|
||||
},
|
||||
{
|
||||
"event": "effect_committed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"version": 1,
|
||||
"variants": 4
|
||||
},
|
||||
"created_at": "2026-11-15 19:47:36.690043+00:00"
|
||||
},
|
||||
{
|
||||
"event": "child_finished",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2740797910000765,
|
||||
"cpu_seconds": 0.2354738329999997,
|
||||
"rss_peak_mib": 79.3125
|
||||
},
|
||||
"created_at": "2026-11-15 19:47:36.847178+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"queued_link_allowed": true,
|
||||
"api_and_broker_restarted": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
PASS: staging_before_handoff_crash
|
||||
PASS: commit_before_publish_and_worker_stopped
|
||||
Traceback (most recent call last):
|
||||
File "<frozen runpy>", line 203, in _run_module_as_main
|
||||
File "<frozen runpy>", line 88, in _run_code
|
||||
File "<repository>/scripts/verify_pipeline.py", line 602, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "<repository>/scripts/verify_pipeline.py", line 318, in main
|
||||
assert restored == ids
|
||||
^^^^^^^^^^^^^^^
|
||||
AssertionError
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
PASS: staging_before_handoff_crash
|
||||
PASS: commit_before_publish_and_worker_stopped
|
||||
PASS: confirmed_broker_entry_survives_sigkill
|
||||
PASS: relay_crash_after_xadd_before_pg_marker
|
||||
PASS: before_effect_sigkill_repeat_1
|
||||
PASS: after_effect_sigkill_repeat_1
|
||||
PASS: after_commit_sigkill_repeat_1
|
||||
PASS: before_effect_sigkill_repeat_2
|
||||
PASS: after_effect_sigkill_repeat_2
|
||||
PASS: after_commit_sigkill_repeat_2
|
||||
PASS: transient_retry_with_jitter
|
||||
PASS: hung_child_killed_and_retried
|
||||
PASS: permanent_failed_registry_and_audited_replay
|
||||
PASS: finite_retry_exhaustion_then_explicit_replay
|
||||
PASS: reordered_and_duplicate_notifications
|
||||
Traceback (most recent call last):
|
||||
File "<frozen runpy>", line 203, in _run_module_as_main
|
||||
File "<frozen runpy>", line 88, in _run_code
|
||||
File "<repository>/scripts/verify_pipeline.py", line 604, in <module>
|
||||
report["worker_metrics"] = inside(
|
||||
|
||||
File "<repository>/scripts/verify_pipeline.py", line 481, in main
|
||||
"reordered_and_duplicate_notifications",
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
...<4 lines>...
|
||||
},
|
||||
^
|
||||
File "<repository>/scripts/verify_pipeline.py", line 48, in wait
|
||||
raise TimeoutError("Fault scenario condition not reached")
|
||||
TimeoutError: Fault scenario condition not reached
|
||||
@@ -0,0 +1,980 @@
|
||||
{
|
||||
"source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88",
|
||||
"date": "2026-11-15T20:03:53.705513+00:00",
|
||||
"track": 2,
|
||||
"scope": "versions",
|
||||
"complete": true,
|
||||
"cases": [
|
||||
{
|
||||
"case": "stale_v1_after_v2_repeat_1",
|
||||
"v1": {
|
||||
"job": {
|
||||
"id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46",
|
||||
"attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd",
|
||||
"pipeline_version": 1,
|
||||
"source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original",
|
||||
"state": "obsolete",
|
||||
"attempt": 1,
|
||||
"cycle": 1,
|
||||
"fence": 2,
|
||||
"lease_owner": null,
|
||||
"lease_until": null,
|
||||
"next_at": "2026-11-15 20:03:59.131799+00:00",
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-15 20:03:59.131799+00:00",
|
||||
"updated_at": "2026-11-15 20:04:25.401001+00:00"
|
||||
},
|
||||
"variants": [],
|
||||
"publications": 0,
|
||||
"physical_keys": [
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/crop",
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/optimized",
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/original",
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/thumbnail"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794773039190-0",
|
||||
"outbox": 43
|
||||
},
|
||||
"created_at": "2026-11-15 20:03:59.200524+00:00"
|
||||
},
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794773054245-0",
|
||||
"outbox": 44
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:14.251008+00:00"
|
||||
},
|
||||
{
|
||||
"event": "version_controller_waiting",
|
||||
"detail": {},
|
||||
"created_at": "2026-11-15 20:04:18.270502+00:00"
|
||||
},
|
||||
{
|
||||
"event": "claimed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"attempt": 1,
|
||||
"version": 1
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:25.403107+00:00"
|
||||
},
|
||||
{
|
||||
"event": "stale_completion_rejected",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"version": 1
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:30.862701+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773039190-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:30.980982+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773054245-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:30.994675+00:00"
|
||||
},
|
||||
{
|
||||
"event": "child_finished",
|
||||
"detail": {
|
||||
"result": "stale"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:31.083095+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079900-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.913386+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079902-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.914038+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079904-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.920903+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079905-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.922028+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079906-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.925707+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079908-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.926905+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079909-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.930906+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079910-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.931649+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079911-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.935325+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079912-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.935997+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079913-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.940184+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773079914-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:39.940743+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"v2": {
|
||||
"job": {
|
||||
"id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9",
|
||||
"attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd",
|
||||
"pipeline_version": 2,
|
||||
"source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original",
|
||||
"state": "ready",
|
||||
"attempt": 1,
|
||||
"cycle": 1,
|
||||
"fence": 1,
|
||||
"lease_owner": null,
|
||||
"lease_until": null,
|
||||
"next_at": "2026-11-15 20:04:28.320656+00:00",
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-15 20:04:28.320656+00:00",
|
||||
"updated_at": "2026-11-15 20:04:30.796918+00:00"
|
||||
},
|
||||
"variants": [
|
||||
{
|
||||
"name": "crop",
|
||||
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop",
|
||||
"sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef",
|
||||
"size_bytes": 1077
|
||||
},
|
||||
{
|
||||
"name": "optimized",
|
||||
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized",
|
||||
"sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced",
|
||||
"size_bytes": 194
|
||||
},
|
||||
{
|
||||
"name": "original",
|
||||
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original",
|
||||
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df",
|
||||
"size_bytes": 742
|
||||
},
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail",
|
||||
"sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199",
|
||||
"size_bytes": 1749
|
||||
}
|
||||
],
|
||||
"publications": 1,
|
||||
"physical_keys": [
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop",
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized",
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original",
|
||||
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"event": "new_version",
|
||||
"detail": {
|
||||
"previous_job": "51c2387d-8bc4-4f37-9ad9-b722464c1f46"
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:28.323557+00:00"
|
||||
},
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794773068363-0",
|
||||
"outbox": 45
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:28.367667+00:00"
|
||||
},
|
||||
{
|
||||
"event": "claimed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"attempt": 1,
|
||||
"version": 2
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:28.373092+00:00"
|
||||
},
|
||||
{
|
||||
"event": "effect_committed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"version": 2,
|
||||
"variants": 4
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:30.798523+00:00"
|
||||
},
|
||||
{
|
||||
"event": "child_finished",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.29709320799975103,
|
||||
"cpu_seconds": 0.24198554100000003,
|
||||
"rss_peak_mib": 79.61328125
|
||||
},
|
||||
"created_at": "2026-11-15 20:04:30.965510+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"new_hashes_unchanged": true
|
||||
},
|
||||
{
|
||||
"case": "stale_v1_after_v2_repeat_2",
|
||||
"v1": {
|
||||
"job": {
|
||||
"id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf",
|
||||
"attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589",
|
||||
"pipeline_version": 1,
|
||||
"source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original",
|
||||
"state": "obsolete",
|
||||
"attempt": 1,
|
||||
"cycle": 1,
|
||||
"fence": 2,
|
||||
"lease_owner": null,
|
||||
"lease_until": null,
|
||||
"next_at": "2026-11-15 20:05:03.917398+00:00",
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-15 20:05:03.917399+00:00",
|
||||
"updated_at": "2026-11-15 20:05:25.442589+00:00"
|
||||
},
|
||||
"variants": [],
|
||||
"publications": 0,
|
||||
"physical_keys": [
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/crop",
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/optimized",
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/original",
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/thumbnail"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794773103958-0",
|
||||
"outbox": 46
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:03.964545+00:00"
|
||||
},
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794773119171-0",
|
||||
"outbox": 47
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:19.176271+00:00"
|
||||
},
|
||||
{
|
||||
"event": "version_controller_waiting",
|
||||
"detail": {},
|
||||
"created_at": "2026-11-15 20:05:22.667065+00:00"
|
||||
},
|
||||
{
|
||||
"event": "claimed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"attempt": 1,
|
||||
"version": 1
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:25.444410+00:00"
|
||||
},
|
||||
{
|
||||
"event": "stale_completion_rejected",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"version": 1
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:30.563412+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773103958-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:30.764786+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773119171-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:30.775876+00:00"
|
||||
},
|
||||
{
|
||||
"event": "child_finished",
|
||||
"detail": {
|
||||
"result": "stale"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:30.815598+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139795-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.811610+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139792-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.811671+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139797-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.818733+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139798-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.819185+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139799-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.824008+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139800-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.824598+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139801-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.829008+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139803-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.829676+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139804-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.833617+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139805-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.834470+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139805-1",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.838465+00:00"
|
||||
},
|
||||
{
|
||||
"event": "duplicate_decision",
|
||||
"detail": {
|
||||
"entry": "1794773139806-0",
|
||||
"state": "obsolete"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:39.839569+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"v2": {
|
||||
"job": {
|
||||
"id": "3c673acd-d8c5-44c7-8366-eed0480acdd8",
|
||||
"attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589",
|
||||
"pipeline_version": 2,
|
||||
"source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original",
|
||||
"state": "ready",
|
||||
"attempt": 1,
|
||||
"cycle": 1,
|
||||
"fence": 1,
|
||||
"lease_owner": null,
|
||||
"lease_until": null,
|
||||
"next_at": "2026-11-15 20:05:28.099769+00:00",
|
||||
"error_code": null,
|
||||
"created_at": "2026-11-15 20:05:28.099769+00:00",
|
||||
"updated_at": "2026-11-15 20:05:30.544288+00:00"
|
||||
},
|
||||
"variants": [
|
||||
{
|
||||
"name": "crop",
|
||||
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop",
|
||||
"sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef",
|
||||
"size_bytes": 1077
|
||||
},
|
||||
{
|
||||
"name": "optimized",
|
||||
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized",
|
||||
"sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced",
|
||||
"size_bytes": 194
|
||||
},
|
||||
{
|
||||
"name": "original",
|
||||
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original",
|
||||
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df",
|
||||
"size_bytes": 742
|
||||
},
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail",
|
||||
"sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199",
|
||||
"size_bytes": 1749
|
||||
}
|
||||
],
|
||||
"publications": 1,
|
||||
"physical_keys": [
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop",
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized",
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original",
|
||||
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"event": "new_version",
|
||||
"detail": {
|
||||
"previous_job": "8679296c-7a76-46b4-a6a4-02b57d0dedaf"
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:28.102749+00:00"
|
||||
},
|
||||
{
|
||||
"event": "published",
|
||||
"detail": {
|
||||
"entry": "1794773128278-0",
|
||||
"outbox": 48
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:28.288692+00:00"
|
||||
},
|
||||
{
|
||||
"event": "claimed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"attempt": 1,
|
||||
"version": 2
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:28.307421+00:00"
|
||||
},
|
||||
{
|
||||
"event": "effect_committed",
|
||||
"detail": {
|
||||
"fence": 1,
|
||||
"version": 2,
|
||||
"variants": 4
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:30.544711+00:00"
|
||||
},
|
||||
{
|
||||
"event": "child_finished",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.28706112500003655,
|
||||
"cpu_seconds": 0.24108328799999978,
|
||||
"rss_peak_mib": 79.671875
|
||||
},
|
||||
"created_at": "2026-11-15 20:05:30.752666+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"new_hashes_unchanged": true
|
||||
},
|
||||
{
|
||||
"case": "malformed_notification_rejected",
|
||||
"durable_diagnostic": true
|
||||
},
|
||||
{
|
||||
"case": "private_iam_and_other_api_download",
|
||||
"security": {
|
||||
"anonymous": 403,
|
||||
"iam": {
|
||||
"create_bucket": "AccessDenied",
|
||||
"write_outside_prefix": "AccessDenied",
|
||||
"list_other_bucket": "AccessDenied"
|
||||
}
|
||||
},
|
||||
"attachment": "5875dfe8-09d9-4ded-9e31-a9329f0974ac",
|
||||
"uploader": "c12dfe7a14a648809e07d709180d70e3",
|
||||
"downloader": "0024aba8755f4f3d92a85c4dbb751f5a",
|
||||
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df"
|
||||
}
|
||||
],
|
||||
"worker_metrics": [
|
||||
{
|
||||
"job_id": "947fae60-fb2b-4000-ac82-76c0aa946f17",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.5540742509999745,
|
||||
"cpu_seconds": 0.47041774699999994,
|
||||
"rss_peak_mib": 79.33203125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "ac292276-9fc0-4d20-b9d0-1c784017f555",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.26326079199998276,
|
||||
"cpu_seconds": 0.23446212599999994,
|
||||
"rss_peak_mib": 79.6796875
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "4d1ee9b7-32e0-44c4-8264-a831b9bc3bdb",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2615961250003238,
|
||||
"cpu_seconds": 0.22782362500000009,
|
||||
"rss_peak_mib": 79.44140625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "1174230c-fb4f-4cdc-9db8-896d002f90a4",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2803810010000234,
|
||||
"cpu_seconds": 0.24079599299999987,
|
||||
"rss_peak_mib": 79.34765625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "4b28f16f-162d-4f60-b1fd-3a907ee834e1",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2567913339998995,
|
||||
"cpu_seconds": 0.228462414,
|
||||
"rss_peak_mib": 79.47265625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "12fa59f7-ed8a-4810-95cf-8b9be4c020e8",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.28690104199995403,
|
||||
"cpu_seconds": 0.23826428900000018,
|
||||
"rss_peak_mib": 79.3046875
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "6247813c-df74-4eeb-9326-cbb13f860336",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.4403786249999939,
|
||||
"cpu_seconds": 0.4114635,
|
||||
"rss_peak_mib": 108.4140625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "9a3fad2f-b3cc-4a28-b996-10c84a4c4b94",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2648974580001777,
|
||||
"cpu_seconds": 0.23192129000000028,
|
||||
"rss_peak_mib": 79.3515625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.30638329200019143,
|
||||
"cpu_seconds": 0.27590770600000014,
|
||||
"rss_peak_mib": 79.4296875
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "cc18c75e-8886-4333-bae8-5da93d626c73",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2798381249999693,
|
||||
"cpu_seconds": 0.24821178899999996,
|
||||
"rss_peak_mib": 79.6640625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "1016e737-7ec1-4fde-9946-a342d70b2a71",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2740797910000765,
|
||||
"cpu_seconds": 0.2354738329999997,
|
||||
"rss_peak_mib": 79.3125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "6e004e09-d91d-42b9-907b-9fda074afeb3",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2604502920003142,
|
||||
"cpu_seconds": 0.22423492300000003,
|
||||
"rss_peak_mib": 79.328125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "817ac0d3-2ae3-4721-a1c4-0d517f624f3a",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2753011250001691,
|
||||
"cpu_seconds": 0.24031520900000003,
|
||||
"rss_peak_mib": 79.31640625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "a68327bb-1d99-4ca1-a541-c197ad0683ea",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2575967509997099,
|
||||
"cpu_seconds": 0.23045399499999997,
|
||||
"rss_peak_mib": 79.84765625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2b40b034-2ea5-4612-8d4e-f099e1e3f65d",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2825338749998991,
|
||||
"cpu_seconds": 0.24960387800000006,
|
||||
"rss_peak_mib": 79.30859375
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "681235f5-558b-44b8-af12-e61d2f6c65bf",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2647618330001933,
|
||||
"cpu_seconds": 0.23189429199999978,
|
||||
"rss_peak_mib": 79.6796875
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "e4b36262-0688-4b3c-90ce-9252e9633353",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2914882079999188,
|
||||
"cpu_seconds": 0.24316999600000022,
|
||||
"rss_peak_mib": 79.80078125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "e1ebdd4b-996f-41d2-aa54-246cc9f4c8d3",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.259311875000094,
|
||||
"cpu_seconds": 0.23085762499999984,
|
||||
"rss_peak_mib": 79.45703125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "fbedc7f8-97a6-4c59-848c-d4ab8045d4d2",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.26234775000011723,
|
||||
"cpu_seconds": 0.22806161800000013,
|
||||
"rss_peak_mib": 79.328125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d",
|
||||
"detail": {
|
||||
"code": "processing_unavailable",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.30820225099978416,
|
||||
"cpu_seconds": 0.27197879400000025,
|
||||
"rss_peak_mib": 79.34765625
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb",
|
||||
"detail": {
|
||||
"code": "worker_timeout",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.26163899999983187,
|
||||
"cpu_seconds": 0.22942266700000014,
|
||||
"rss_peak_mib": 79.33203125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1",
|
||||
"detail": {
|
||||
"code": "invalid_image",
|
||||
"result": "permanent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.27349454200020773,
|
||||
"cpu_seconds": 0.23941921199999983,
|
||||
"rss_peak_mib": 79.28125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"detail": {
|
||||
"code": "processing_unavailable",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"detail": {
|
||||
"code": "processing_unavailable",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"detail": {
|
||||
"code": "processing_unavailable",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"detail": {
|
||||
"code": "processing_unavailable",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"detail": {
|
||||
"code": "processing_unavailable",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.2960601250001673,
|
||||
"cpu_seconds": 0.26317221,
|
||||
"rss_peak_mib": 79.25
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "20c34eb3-0582-4144-847c-fa037647f863",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.3045327089998864,
|
||||
"cpu_seconds": 0.2630334579999998,
|
||||
"rss_peak_mib": 79.578125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "c013ea06-4dea-4e65-b715-8a3f5976fa57",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.3056550410001364,
|
||||
"cpu_seconds": 0.26324616899999986,
|
||||
"rss_peak_mib": 79.65234375
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "44de0dbe-f2b8-40bd-8b65-479e8f7243b5",
|
||||
"detail": {
|
||||
"code": "worker_timeout",
|
||||
"result": "transient"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.26621149999982663,
|
||||
"cpu_seconds": 0.23299866699999994,
|
||||
"rss_peak_mib": 79.67578125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.29709320799975103,
|
||||
"cpu_seconds": 0.24198554100000003,
|
||||
"rss_peak_mib": 79.61328125
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46",
|
||||
"detail": {
|
||||
"result": "stale"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.28706112500003655,
|
||||
"cpu_seconds": 0.24108328799999978,
|
||||
"rss_peak_mib": 79.671875
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf",
|
||||
"detail": {
|
||||
"result": "stale"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "8eedc435-5509-4724-9a18-e478d920f101",
|
||||
"detail": {
|
||||
"result": "ready",
|
||||
"seconds": 0.32150650000039604,
|
||||
"cpu_seconds": 0.28338829200000015,
|
||||
"rss_peak_mib": 79.37109375
|
||||
}
|
||||
}
|
||||
],
|
||||
"audit": [
|
||||
{
|
||||
"job_id": "cc18c75e-8886-4333-bae8-5da93d626c73",
|
||||
"action": "new_version",
|
||||
"operator": "synthetic-verifier",
|
||||
"reason": "Explicit fault matrix replay",
|
||||
"created_at": "2026-11-15 19:46:27.434253+00:00"
|
||||
},
|
||||
{
|
||||
"job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1",
|
||||
"action": "replay",
|
||||
"operator": "synthetic-verifier",
|
||||
"reason": "Explicit fault matrix replay",
|
||||
"created_at": "2026-11-15 19:56:33.964020+00:00"
|
||||
},
|
||||
{
|
||||
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
|
||||
"action": "replay",
|
||||
"operator": "synthetic-verifier",
|
||||
"reason": "Explicit fault matrix replay",
|
||||
"created_at": "2026-11-15 19:57:37.397099+00:00"
|
||||
},
|
||||
{
|
||||
"job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9",
|
||||
"action": "new_version",
|
||||
"operator": "synthetic-verifier",
|
||||
"reason": "Explicit fault matrix replay",
|
||||
"created_at": "2026-11-15 19:59:07.104714+00:00"
|
||||
},
|
||||
{
|
||||
"job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9",
|
||||
"action": "new_version",
|
||||
"operator": "synthetic-verifier",
|
||||
"reason": "Held stale writer race",
|
||||
"created_at": "2026-11-15 20:04:28.322925+00:00"
|
||||
},
|
||||
{
|
||||
"job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8",
|
||||
"action": "new_version",
|
||||
"operator": "synthetic-verifier",
|
||||
"reason": "Held stale writer race",
|
||||
"created_at": "2026-11-15 20:05:28.102142+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
+13
-4
@@ -1,7 +1,16 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура ЛР5
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Топология и реальные pinned versions — compose.yaml; сборка MinIO/mc из
|
||||
закреплённых commits — s3.Dockerfile; bucket/IAM — s3-bootstrap.sh и s3-policy.json.
|
||||
Credentials создаёт scripts/bootstrap.py, значения не входят в Git/image.
|
||||
API/worker/reconciler получают scoped application keys, relay только PG+broker,
|
||||
root keys доступны только S3/bootstrap. Консоль S3 слушает loopback контейнера.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
Сервисы: postgres persistent volume; cache disposable; s3 persistent volume;
|
||||
broker отдельный Valkey Streams persistent AOF volume; relay/outbox/watchdog;
|
||||
worker отдельный2CPU1536MiB2slots; api2replicas; lb один localhost:8085.
|
||||
S3/broker/PG/API host ports не публикуются. Сеть — private Compose default;
|
||||
production TLS/изоляция от потери host не заявлены. Детали — REPORT.md/SCHEMA.md.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
compose.test.yaml включает fault instrumentation и отдельный controller,
|
||||
без публичных endpoints; обычный запуск не имеет этого controller/hooks.
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
|
||||
// go mod download validates its h1 checksum through the Go checksum database.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const version = "v0.0.0-20251015172955-9e49d5e7a648"
|
||||
const total int64 = 25342606
|
||||
const chunk int64 = 512 * 1024
|
||||
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
|
||||
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
|
||||
if err := os.MkdirAll(directory, 0755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
target := filepath.Join(directory, version+".zip")
|
||||
if info, err := os.Stat(target); err == nil && info.Size() == total {
|
||||
fmt.Println("Source ZIP cached; Go will validate checksum")
|
||||
return
|
||||
}
|
||||
client := &http.Client{Timeout: 90 * time.Second}
|
||||
jobs := make(chan int64)
|
||||
errors := make(chan error, 64)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < 8; worker++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for start := range jobs {
|
||||
end := min(start+chunk, total) - 1
|
||||
part := fmt.Sprintf("%s.part-%d", target, start)
|
||||
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
|
||||
continue
|
||||
}
|
||||
var last error
|
||||
for attempt := 0; attempt < 4; attempt++ {
|
||||
req, _ := http.NewRequest("GET", source, nil)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
last = fmt.Errorf("source range request failed")
|
||||
continue
|
||||
}
|
||||
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
|
||||
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
|
||||
response.Body.Close()
|
||||
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
|
||||
continue
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
|
||||
response.Body.Close()
|
||||
if err != nil || int64(len(data)) != end-start+1 {
|
||||
last = fmt.Errorf("incomplete source range")
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(part, data, 0644); err != nil {
|
||||
last = err
|
||||
continue
|
||||
}
|
||||
last = nil
|
||||
fmt.Printf("Source part %d verified\n", start/chunk+1)
|
||||
break
|
||||
}
|
||||
if last != nil {
|
||||
errors <- last
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for start := int64(0); start < total; start += chunk {
|
||||
jobs <- start
|
||||
}
|
||||
close(jobs)
|
||||
group.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
panic(err)
|
||||
}
|
||||
out, err := os.Create(target + ".partial")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for start := int64(0); start < total; start += chunk {
|
||||
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := io.Copy(out, part); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
part.Close()
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := os.Rename(target+".partial", target); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 512
|
||||
stats socket /tmp/admin.sock mode 600 level admin
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 15s
|
||||
timeout server 8s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
|
||||
server-template api 1-8 api:8080
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
root_user=$(cat /run/secrets/s3_root_user)
|
||||
root_password=$(cat /run/secrets/s3_root_password)
|
||||
api_key=$(cat /run/secrets/s3_access_key)
|
||||
api_secret=$(cat /run/secrets/s3_secret_key)
|
||||
# Credentials are passed only inside this isolated bootstrap container.
|
||||
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
|
||||
mc mb --ignore-existing local/messenger >/dev/null 2>&1
|
||||
mc anonymous set none local/messenger >/dev/null 2>&1
|
||||
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
|
||||
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
|
||||
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
|
||||
printf '%s\n' 'Private bucket and scoped application policy ready'
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001
|
||||
exec minio server /data --address :9000 --console-address 127.0.0.1:9001
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger/objects/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
|
||||
|
||||
FROM toolchain AS minio-build
|
||||
WORKDIR /source
|
||||
COPY infra/fetch-source.go /tmp/fetch-source.go
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
go run /tmp/fetch-source.go && \
|
||||
for attempt in 1 2 3 4; do \
|
||||
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
|
||||
test "$attempt" -lt 4 || exit 1; sleep 1; \
|
||||
done && \
|
||||
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
for attempt in 1 2 3; do \
|
||||
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
|
||||
test "$attempt" -lt 3 || exit 1; sleep 1; \
|
||||
done
|
||||
|
||||
FROM toolchain AS mc-build
|
||||
WORKDIR /source
|
||||
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
|
||||
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
go build -trimpath -ldflags='-s -w' -o /out/mc .
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
|
||||
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
|
||||
COPY --from=minio-build /out/minio /usr/local/bin/minio
|
||||
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
|
||||
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
|
||||
RUN mkdir /data && chown 10001:10001 /data
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
|
||||
EXPOSE 9000
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
|
||||
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
|
||||
COPY --from=mc-build /out/mc /usr/local/bin/mc
|
||||
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
|
||||
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
|
||||
COPY infra/s3-policy.json /policy.json
|
||||
USER 10001:10001
|
||||
ENV MC_CONFIG_DIR=/tmp/mc
|
||||
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL,
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY(user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
|
||||
@@ -0,0 +1,32 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||
ALTER TABLE messages ADD UNIQUE(chat_id,id);
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
|
||||
error_code text,
|
||||
uploader_instance text NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE(chat_id,id),
|
||||
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(message_id,attachment_id),
|
||||
UNIQUE(message_id,position),
|
||||
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
|
||||
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
|
||||
);
|
||||
@@ -0,0 +1,54 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||||
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
|
||||
ALTER TABLE attachments ADD COLUMN published_version integer;
|
||||
CREATE TABLE upload_reservations (
|
||||
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
|
||||
);
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
|
||||
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
|
||||
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
|
||||
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
|
||||
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
|
||||
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz
|
||||
);
|
||||
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
|
||||
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
|
||||
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL, sha256 text NOT NULL,
|
||||
width integer, height integer,
|
||||
PRIMARY KEY(attachment_id,pipeline_version,name)
|
||||
);
|
||||
CREATE TABLE effect_publications (
|
||||
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
|
||||
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
|
||||
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE TABLE pipeline_audit (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
|
||||
operator text NOT NULL, reason text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
|
||||
CREATE TABLE pipeline_faults (
|
||||
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
|
||||
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
|
||||
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
|
||||
);
|
||||
CREATE TABLE pipeline_events (
|
||||
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
|
||||
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,45 @@
|
||||
[project]
|
||||
name = "mtusi-messenger-lab5"
|
||||
version = "1.0.0"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"fastapi==0.143.3",
|
||||
"pydantic==2.14.1",
|
||||
"uvicorn==0.54.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"argon2-cffi>=25,<26",
|
||||
"redis==8.1.0",
|
||||
"minio==7.2.20",
|
||||
"pillow==13.0.0",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validator==0.7.2", "psutil>=7,<8", "matplotlib>=3.10,<4"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 79
|
||||
target-version = "py314"
|
||||
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
|
||||
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"research/*.py" = ["E501"]
|
||||
"scripts/*.py" = ["E501"]
|
||||
|
||||
# Keep SQL statements together; the formatter controls executable statements.
|
||||
"src/attachments.py" = ["E501"]
|
||||
"src/pipeline.py" = ["E501"]
|
||||
"src/worker.py" = ["E501"]
|
||||
"src/operator.py" = ["E501"]
|
||||
"src/reconcile.py" = ["E501"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra"
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Summarize measured fault invariants and child resource samples."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def effects(case):
|
||||
for key in ["effect", "v1", "v2"]:
|
||||
if key in case:
|
||||
yield key, case[key]
|
||||
for i, item in enumerate(case.get("effects", [])):
|
||||
yield str(i + 1), item
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("input", type=Path)
|
||||
p.add_argument("--output", required=True, type=Path)
|
||||
a = p.parse_args()
|
||||
d = json.loads(a.input.read_text())
|
||||
assert d["complete"] and d.get("scope", "all") == "all"
|
||||
rows = []
|
||||
for case in d["cases"]:
|
||||
for role, e in effects(case):
|
||||
j = e["job"]
|
||||
rows.append(
|
||||
dict(
|
||||
case=case["case"],
|
||||
role=role,
|
||||
job_id=j["id"],
|
||||
version=j["pipeline_version"],
|
||||
attempt=j["attempt"],
|
||||
cycle=j["cycle"],
|
||||
fence=j["fence"],
|
||||
state=j["state"],
|
||||
publications=e["publications"],
|
||||
variant_rows=len(e["variants"]),
|
||||
physical_keys=len(e["physical_keys"]),
|
||||
)
|
||||
)
|
||||
a.output.mkdir(parents=True, exist_ok=True)
|
||||
with (a.output / "effects.csv").open("w") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
ready = [r for r in rows if r["state"] == "ready"]
|
||||
assert all(
|
||||
r["publications"] == 1
|
||||
and r["variant_rows"] == 4
|
||||
and r["physical_keys"] == 4
|
||||
for r in ready
|
||||
)
|
||||
stale = [r for r in rows if r["state"] == "obsolete"]
|
||||
assert all(r["publications"] == 0 for r in stale)
|
||||
job_ids = {r["job_id"] for r in rows}
|
||||
samples = [
|
||||
x["detail"]
|
||||
for x in d["worker_metrics"]
|
||||
if str(x["job_id"]) in job_ids
|
||||
and x["detail"].get("result") == "ready"
|
||||
and "rss_peak_mib" in x["detail"]
|
||||
]
|
||||
stats = {
|
||||
"source_commit": d["source_commit"],
|
||||
"cases": len(d["cases"]),
|
||||
"observed_ready_effects": len(ready),
|
||||
"observed_obsolete_effects": len(stale),
|
||||
"all_ready_have_one_publication_and_four_variants": True,
|
||||
"all_obsolete_have_zero_publications": True,
|
||||
"successful_child_samples": len(samples),
|
||||
"child_elapsed_range_seconds": [
|
||||
min(x["seconds"] for x in samples),
|
||||
max(x["seconds"] for x in samples),
|
||||
],
|
||||
"child_cpu_range_seconds": [
|
||||
min(x["cpu_seconds"] for x in samples),
|
||||
max(x["cpu_seconds"] for x in samples),
|
||||
],
|
||||
"child_peak_rss_range_mib": [
|
||||
min(x["rss_peak_mib"] for x in samples),
|
||||
max(x["rss_peak_mib"] for x in samples),
|
||||
],
|
||||
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
|
||||
}
|
||||
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
|
||||
axes[0].bar(
|
||||
range(1, len(rows) + 1),
|
||||
[r["publications"] for r in rows],
|
||||
color=[
|
||||
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
|
||||
],
|
||||
)
|
||||
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
|
||||
axes[0].set(
|
||||
xlabel="Effect observation (CSV row)",
|
||||
ylabel="Committed publications",
|
||||
title="One publication per ready version; stale v1 = 0",
|
||||
yticks=[0, 1],
|
||||
)
|
||||
axes[1].scatter(
|
||||
[x["seconds"] for x in samples],
|
||||
[x["rss_peak_mib"] for x in samples],
|
||||
color="#246b8e",
|
||||
s=35,
|
||||
alpha=0.8,
|
||||
)
|
||||
axes[1].set(
|
||||
xlabel="Child elapsed, seconds",
|
||||
ylabel="Child peak RSS, MiB",
|
||||
title=f"{len(samples)} completed child samples (synthetic320×200)",
|
||||
)
|
||||
for ax in axes:
|
||||
ax.grid(axis="y", alpha=0.2)
|
||||
fig.savefig(a.output / "pipeline-results.png", dpi=150)
|
||||
plt.close(fig)
|
||||
print(json.dumps(stats, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def create(path, value):
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
return
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
stream.write(value + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
pg = directory / "postgres_password"
|
||||
cache = directory / "cache_password"
|
||||
create(pg, secrets.token_urlsafe(32))
|
||||
create(cache, secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "database_url",
|
||||
"postgresql://messenger:"
|
||||
+ pg.read_text().strip()
|
||||
+ "@postgres:5432/messenger",
|
||||
)
|
||||
create(
|
||||
directory / "cache_url",
|
||||
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "valkey.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
|
||||
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
|
||||
+ cache.read_text().strip(),
|
||||
)
|
||||
create(directory / "s3_root_user", secrets.token_hex(10))
|
||||
create(directory / "s3_root_password", secrets.token_urlsafe(32))
|
||||
create(directory / "s3_access_key", secrets.token_hex(10))
|
||||
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
|
||||
create(directory / "broker_password", secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "broker_url",
|
||||
"redis://:"
|
||||
+ (directory / "broker_password").read_text().strip()
|
||||
+ "@broker:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "broker.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
|
||||
+ (directory / "broker_password").read_text().strip(),
|
||||
)
|
||||
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
|
||||
print("Local configuration ready; existing files preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Явные операторские действия только над API текущего Compose-проекта."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def containers():
|
||||
ids = command("docker", "compose", "ps", "-q", "api").split()
|
||||
return json.loads(command("docker", "inspect", *ids)) if ids else []
|
||||
|
||||
|
||||
def alias(cid):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=2); "
|
||||
"print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
return command("docker", "exec", cid, "python", "-c", code)
|
||||
|
||||
|
||||
def wait_count(count):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
data = containers()
|
||||
if len(data) == count and all(
|
||||
c["State"].get("Health", {}).get("Status") == "healthy"
|
||||
for c in data
|
||||
):
|
||||
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
|
||||
time.sleep(3)
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("Replica readiness")
|
||||
|
||||
|
||||
def main():
|
||||
action = sys.argv[1]
|
||||
data = containers()
|
||||
if action == "stop":
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [c for c in data if alias(c["Id"]) == target]
|
||||
if len(matches) != 1 or len(data) < 2:
|
||||
raise RuntimeError("Target must identify one running replica")
|
||||
command("docker", "stop", "--time", "10", matches[0]["Id"])
|
||||
print("Selected API stopped")
|
||||
elif action == "rolling":
|
||||
count = len(data)
|
||||
if count < 2:
|
||||
raise RuntimeError("Need at least two replicas")
|
||||
# Сначала добавляется готовая реплика, затем удаляется одна старая.
|
||||
for old in data:
|
||||
command(
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--scale",
|
||||
f"api={count + 1}",
|
||||
"api",
|
||||
)
|
||||
wait_count(count + 1)
|
||||
command("docker", "stop", "--time", "10", old["Id"])
|
||||
command("docker", "rm", old["Id"])
|
||||
wait_count(count)
|
||||
print("Sequential replacement completed")
|
||||
else:
|
||||
raise ValueError("Unknown action")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose restart api postgres cache s3 broker relay worker lb
|
||||
docker compose up -d --no-build --no-recreate --wait
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py rolling
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8085")
|
||||
|
||||
|
||||
def command(args, *, input=None):
|
||||
result = subprocess.run(
|
||||
args, cwd=ROOT, input=input, capture_output=True, text=True
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(
|
||||
"Command failed: "
|
||||
+ args[0]
|
||||
+ "; "
|
||||
+ result.stderr.replace(str(ROOT), "<repository>")
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command(["docker", "compose", *args])
|
||||
|
||||
|
||||
def sql(query, *, database="messenger"):
|
||||
return compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
database,
|
||||
"-A",
|
||||
"-t",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-c",
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def cache(command_name, *args):
|
||||
script = (
|
||||
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
|
||||
'exec valkey-cli "$@"'
|
||||
)
|
||||
return compose(
|
||||
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
|
||||
)
|
||||
|
||||
|
||||
def ready(base=BASE, *, timeout=30):
|
||||
end = time.monotonic() + timeout
|
||||
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
if client.get("/health/ready").status_code == 200:
|
||||
return
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError("API readiness")
|
||||
|
||||
|
||||
def actor(client):
|
||||
username = "verify_" + uuid4().hex[:20]
|
||||
password = "Synthetic-" + uuid4().hex + ":🌍"
|
||||
result = client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": username,
|
||||
"display_name": "Synthetic",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
user = result.json()
|
||||
session = client.post(
|
||||
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
|
||||
)
|
||||
assert session.status_code == 201
|
||||
return user, password, session.json()
|
||||
|
||||
|
||||
def auth(session):
|
||||
return {"Authorization": "Bearer " + session["token"]}
|
||||
|
||||
|
||||
def extra_api(*, ttl=3600, pool_size=1):
|
||||
name = "mtusi-lab5-test-" + uuid4().hex[:10]
|
||||
compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"-e",
|
||||
"SESSION_TTL_SECONDS=" + str(ttl),
|
||||
"-e",
|
||||
"DB_POOL_SIZE=" + str(pool_size),
|
||||
"-p",
|
||||
"127.0.0.1::8080",
|
||||
"api",
|
||||
)
|
||||
try:
|
||||
info = json.loads(command(["docker", "inspect", name]))[0]
|
||||
base = (
|
||||
"http://127.0.0.1:"
|
||||
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
|
||||
)
|
||||
ready(base)
|
||||
return name, base
|
||||
except Exception:
|
||||
command(["docker", "rm", "-f", name])
|
||||
raise
|
||||
|
||||
|
||||
def instances():
|
||||
data = {}
|
||||
for role in ["api", "postgres", "cache"]:
|
||||
cid = compose("ps", "-q", role)
|
||||
info = json.loads(command(["docker", "inspect", cid]))[0]
|
||||
data[role] = {
|
||||
"container": cid[:12],
|
||||
"pid": info["State"]["Pid"],
|
||||
"started_at": info["State"]["StartedAt"],
|
||||
}
|
||||
return data
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py stop
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose stop --timeout 1 worker
|
||||
@@ -0,0 +1,254 @@
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROGRESS = {}
|
||||
|
||||
|
||||
def save_incomplete():
|
||||
receipt = PROGRESS.get("receipt")
|
||||
if receipt and receipt.get("result") != "PASS":
|
||||
target = PROGRESS["output"].with_name(
|
||||
PROGRESS["output"].stem + "-failed.json"
|
||||
)
|
||||
if target.exists():
|
||||
target = target.with_name(
|
||||
target.stem + "-" + uuid4().hex[:8] + target.suffix
|
||||
)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
{**receipt, "result": "INCOMPLETE"},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
atexit.register(save_incomplete)
|
||||
|
||||
|
||||
def run(args, *, cwd, env, expected=0):
|
||||
result = subprocess.run(
|
||||
args, cwd=cwd, env=env, capture_output=True, text=True
|
||||
)
|
||||
|
||||
def redact(text):
|
||||
return (
|
||||
text.replace(str(cwd), "<clean-clone>")
|
||||
.replace(str(ROOT), "<repository>")
|
||||
.replace(str(Path.home()), "<home>")
|
||||
)
|
||||
|
||||
if result.returncode != expected:
|
||||
if "receipt" in PROGRESS:
|
||||
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
|
||||
raise RuntimeError(redact(result.stdout + result.stderr))
|
||||
return redact(result.stdout.strip())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/clean-clone.json")
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise SystemExit("Choose new output path")
|
||||
assert not subprocess.check_output(
|
||||
["git", "status", "--porcelain"], cwd=ROOT
|
||||
)
|
||||
sha = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
||||
).strip()
|
||||
identity = uuid4().hex[:10]
|
||||
project = "mtusi-lab5-clean-" + identity
|
||||
with socket.socket() as reservation:
|
||||
reservation.bind(("127.0.0.1", 0))
|
||||
public_port = str(reservation.getsockname()[1])
|
||||
env = {
|
||||
**os.environ,
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"API_IMAGE": "mtusi-messenger:lab5-clean-" + identity,
|
||||
"PUBLIC_PORT": public_port,
|
||||
"VCS_REF": sha,
|
||||
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
receipt = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": sha,
|
||||
"fresh_secrets_environment_and_volumes": True,
|
||||
"command": "python scripts/verify_clean.py",
|
||||
"fixed_host_port_for_restart": int(public_port),
|
||||
}
|
||||
PROGRESS.update(receipt=receipt, output=output)
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab5-clean-") as temporary:
|
||||
clone = Path(temporary) / "lab5"
|
||||
run(
|
||||
[
|
||||
"git",
|
||||
"clone",
|
||||
"--no-hardlinks",
|
||||
"--branch",
|
||||
"solution/bvt2301-maslov",
|
||||
str(ROOT),
|
||||
str(clone),
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
)
|
||||
for name in [".env", ".venv", "secrets"]:
|
||||
assert not (clone / name).exists()
|
||||
run(["uv", "sync", "--locked"], cwd=clone, env=env)
|
||||
receipt["static_checks"] = run(
|
||||
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
|
||||
run(["make", "bootstrap"], cwd=clone, env=env)
|
||||
try:
|
||||
run(
|
||||
["docker", "compose", "up", "-d", "--build", "--wait"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
env["BASE_URL"] = "http://127.0.0.1:" + public_port
|
||||
receipt["smoke"] = run(
|
||||
["make", "test", "BASE_URL=" + env["BASE_URL"]],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["http_tests"] = run(
|
||||
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
|
||||
)
|
||||
run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cluster",
|
||||
"--output",
|
||||
"evidence/clean-cluster.json",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
for command_args in [
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"seed",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_pipeline",
|
||||
"--output",
|
||||
"evidence/clean-pipeline.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"verify",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
"--output",
|
||||
"evidence/clean-cleanup.json",
|
||||
],
|
||||
]:
|
||||
run(command_args, cwd=clone, env=env)
|
||||
receipt["pipeline"] = json.loads(
|
||||
(clone / "evidence/clean-pipeline.json").read_text()
|
||||
)
|
||||
receipt["cleanup"] = json.loads(
|
||||
(clone / "evidence/clean-cleanup.json").read_text()
|
||||
)
|
||||
receipt["cluster"] = json.loads(
|
||||
(clone / "evidence/clean-cluster.json").read_text()
|
||||
)
|
||||
import hashlib
|
||||
|
||||
expected = {
|
||||
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in (clone / "src").glob("*.py")
|
||||
}
|
||||
ids = run(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
|
||||
).split()
|
||||
assert len(ids) == 2
|
||||
code = (
|
||||
"import hashlib,json,pathlib; print(json.dumps({p.name:"
|
||||
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
|
||||
)
|
||||
ids += run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"ps",
|
||||
"-q",
|
||||
"worker",
|
||||
"relay",
|
||||
"reconciler",
|
||||
"controller",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
).split()
|
||||
assert len(ids) == 6
|
||||
for cid in ids:
|
||||
actual = json.loads(
|
||||
run(
|
||||
["docker", "exec", cid, "python", "-c", code],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
)
|
||||
assert actual == expected
|
||||
info = json.loads(
|
||||
run(["docker", "inspect", cid], cwd=clone, env=env)
|
||||
)[0]
|
||||
assert (
|
||||
info["Config"]["Labels"][
|
||||
"org.opencontainers.image.revision"
|
||||
]
|
||||
== sha
|
||||
)
|
||||
assert not info["HostConfig"]["PortBindings"]
|
||||
receipt[
|
||||
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
|
||||
] = True
|
||||
receipt["fsck"] = run(
|
||||
["git", "fsck", "--full"], cwd=clone, env=env
|
||||
)
|
||||
receipt["checks_completed"] = True
|
||||
finally:
|
||||
assert project.startswith("mtusi-lab5-clean-")
|
||||
run(
|
||||
["docker", "compose", "down", "--volumes", "--remove-orphans"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["result"] = "PASS"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Real S3 retention test: seed is separate from verification output."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
from scripts.verify_pipeline import inside, operator, wait
|
||||
|
||||
|
||||
def seed(path):
|
||||
if path.exists():
|
||||
raise ValueError("Choose a new seed file")
|
||||
ready()
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
|
||||
user, _, session = actor(c)
|
||||
headers = auth(session)
|
||||
result = c.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Cleanup synthetic seed",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
chat = result.json()["id"]
|
||||
r = c.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**headers, "Content-Type": "image/png"},
|
||||
params={"filename": "retained.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202
|
||||
attachment = r.json()["id"]
|
||||
wait(
|
||||
lambda: (
|
||||
c.get(
|
||||
"/api/v1/attachments/" + attachment, headers=headers
|
||||
).json()["status"]
|
||||
== "ready"
|
||||
)
|
||||
)
|
||||
job = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
data = {
|
||||
"seed_id": uuid4().hex,
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"ready_attachment": attachment,
|
||||
"ready_job": job,
|
||||
"chat": chat,
|
||||
"owner": user["id"],
|
||||
}
|
||||
# Real old v1 objects will become obsolete after v2 is ready.
|
||||
second = operator(job, version=2)
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
|
||||
)
|
||||
)
|
||||
data["version2_job"] = second
|
||||
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
|
||||
data["failed_id"] = str(uuid4())
|
||||
data["active_id"] = str(uuid4())
|
||||
data["reservation_id"] = str(uuid4())
|
||||
data["expired_reservation_id"] = str(uuid4())
|
||||
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
|
||||
data["active_source"] = f"objects/staging/{data['active_id']}/original"
|
||||
data["reservation_key"] = (
|
||||
f"objects/staging/{data['reservation_id']}/original"
|
||||
)
|
||||
data["expired_reservation_key"] = (
|
||||
f"objects/staging/{data['expired_reservation_id']}/original"
|
||||
)
|
||||
inside(f"""from pathlib import Path
|
||||
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
|
||||
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
|
||||
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
|
||||
with db.connection() as c, c.transaction():
|
||||
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
|
||||
key='objects/staging/'+identity+'/original'
|
||||
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
|
||||
print('true')""")
|
||||
data["seeded_unix"] = time.time()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: retention seed created; real S3 age must reach300s", flush=True
|
||||
)
|
||||
|
||||
|
||||
def verify(seed_path, output):
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
data = json.loads(seed_path.read_text())
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"seed": seed_path.name,
|
||||
"complete": False,
|
||||
}
|
||||
try:
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
|
||||
def age():
|
||||
return inside(
|
||||
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
|
||||
)
|
||||
|
||||
while age() < 302:
|
||||
time.sleep(min(5, 303 - age()))
|
||||
report["real_s3_age_seconds"] = age()
|
||||
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
|
||||
inside(
|
||||
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
|
||||
)
|
||||
oldkeys = [
|
||||
data["orphan"],
|
||||
data["failed_source"],
|
||||
data["expired_reservation_key"],
|
||||
] + [
|
||||
f"objects/{data['ready_attachment']}/v1/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
protected = [data["active_source"], data["reservation_key"], fresh] + [
|
||||
f"objects/{data['ready_attachment']}/v2/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
before = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(before.values())
|
||||
dry = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
|
||||
)
|
||||
candidates = {a["key"] for a in dry["actions"]}
|
||||
assert set(oldkeys) <= candidates and not set(protected) & candidates
|
||||
afterdry = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert afterdry == before
|
||||
applied = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
repeat = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
assert not repeat["actions"]
|
||||
after = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(not after[k] for k in oldkeys) and all(
|
||||
after[k] for k in protected
|
||||
)
|
||||
failed = inside(
|
||||
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
|
||||
)
|
||||
assert failed["error_code"] == "source_expired"
|
||||
try:
|
||||
operator(failed["id"])
|
||||
except RuntimeError:
|
||||
replay_refused = True
|
||||
else:
|
||||
raise AssertionError("Expired source replay must fail")
|
||||
# Expired reservation was removed, so a delayed API cannot commit its handoff.
|
||||
fenced = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
|
||||
)
|
||||
assert fenced
|
||||
report.update(
|
||||
dry_run=dry,
|
||||
apply=applied,
|
||||
repeat=repeat,
|
||||
before=before,
|
||||
after=after,
|
||||
dry_run_unchanged=True,
|
||||
active_current_and_fresh_protected=True,
|
||||
expired_source_replay_refused=replay_refused,
|
||||
expired_writer_fenced=fenced,
|
||||
complete=True,
|
||||
)
|
||||
finally:
|
||||
# Remove only this fixture's artificial active reservation/job; never real jobs.
|
||||
inside(
|
||||
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["seed", "verify"])
|
||||
p.add_argument("--seed", required=True, type=Path)
|
||||
p.add_argument("--output", type=Path)
|
||||
a = p.parse_args()
|
||||
if a.action == "seed":
|
||||
seed(a.seed)
|
||||
else:
|
||||
if a.output is None:
|
||||
p.error("output required")
|
||||
verify(a.seed, a.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,347 @@
|
||||
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts import cluster
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = compose("ps", "-q").split()
|
||||
result = []
|
||||
for c in json.loads(command(["docker", "inspect", *ids])):
|
||||
role = c["Config"]["Labels"]["com.docker.compose.service"]
|
||||
entry = {
|
||||
"role": role,
|
||||
"container": c["Id"][:12],
|
||||
"pid": c["State"]["Pid"],
|
||||
"started_at": c["State"]["StartedAt"],
|
||||
"ports": c["HostConfig"]["PortBindings"],
|
||||
"cpu_nano": c["HostConfig"]["NanoCpus"],
|
||||
"memory_limit": c["HostConfig"]["Memory"],
|
||||
}
|
||||
if role == "api":
|
||||
entry["instance"] = cluster.alias(c["Id"])
|
||||
assert not entry["ports"]
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
|
||||
def exercise(client, session, chat, action, *, seconds=15):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
observations, confirmed, errors, reads = [], {}, 0, 0
|
||||
started = time.monotonic()
|
||||
finished = threading.Event()
|
||||
action_result = {}
|
||||
|
||||
def operate():
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
action_result["error"] = type(exc).__name__
|
||||
finally:
|
||||
action_result["finished"] = round(time.monotonic() - started, 3)
|
||||
finished.set()
|
||||
|
||||
thread = threading.Thread(target=operate)
|
||||
thread.start()
|
||||
key = uuid4().hex
|
||||
try:
|
||||
while (
|
||||
time.monotonic() - started < seconds
|
||||
or not finished.is_set()
|
||||
or time.monotonic() - started
|
||||
< action_result.get("finished", 0) + 2
|
||||
):
|
||||
h = {**auth(session), "Idempotency-Key": key}
|
||||
try:
|
||||
r = client.post(path, headers=h, json={"text": key})
|
||||
if r.status_code != 201:
|
||||
errors += 1
|
||||
continue
|
||||
m = r.json()
|
||||
confirmed[m["id"]] = m
|
||||
observations.append(
|
||||
{
|
||||
"at": round(time.monotonic() - started, 3),
|
||||
"instance": r.headers["x-instance-id"],
|
||||
}
|
||||
)
|
||||
key = uuid4().hex
|
||||
read = client.get(
|
||||
path, headers=auth(session), params={"limit": 100}
|
||||
)
|
||||
reads += read.status_code == 200
|
||||
errors += read.status_code != 200
|
||||
except httpx.HTTPError:
|
||||
errors += 1
|
||||
time.sleep(0.04)
|
||||
finally:
|
||||
thread.join(timeout=90)
|
||||
assert not action_result.get("error"), action_result
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
r = client.get(
|
||||
path,
|
||||
headers=auth(session),
|
||||
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
page = r.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m["id"]: m for m in found}
|
||||
assert all(by_id.get(k) == v for k, v in confirmed.items())
|
||||
texts = [m["text"] for m in found]
|
||||
assert len(texts) == len(set(texts))
|
||||
recovery = next(
|
||||
(
|
||||
x["at"]
|
||||
for x in observations
|
||||
if x["at"] >= action_result["finished"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert recovery is not None and (seconds > 20 or recovery <= 10)
|
||||
elapsed = time.monotonic() - started
|
||||
return {
|
||||
"confirmed_writes": len(confirmed),
|
||||
"successful_reads": reads,
|
||||
"errors": errors,
|
||||
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
|
||||
"first_success_after_action_seconds": recovery,
|
||||
"action": action_result,
|
||||
"observations": observations,
|
||||
"confirmed_preserved": True,
|
||||
"duplicate_texts": False,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/cluster-repeat.json")
|
||||
args = parser.parse_args()
|
||||
report = {
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ready()
|
||||
initial = snapshot()
|
||||
apis = [c for c in initial if c["role"] == "api"]
|
||||
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
|
||||
report["before"] = initial
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
|
||||
user, _, session = actor(client)
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={
|
||||
"title": "Cluster verification",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
chat = r.json()
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
|
||||
def repeat(_):
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=8
|
||||
) as c:
|
||||
r = c.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": key},
|
||||
json={"text": "Same concurrent request"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["x-instance-id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
repeated = list(pool.map(repeat, range(30)))
|
||||
assert len({m[0]["id"] for m in repeated}) == 1
|
||||
assert len({m[1] for m in repeated}) == 2
|
||||
report["parallel_replay"] = {
|
||||
"requests": 30,
|
||||
"messages": 1,
|
||||
"instances": sorted({m[1] for m in repeated}),
|
||||
}
|
||||
# Сбой второй записи транзакции не должен оставить сообщение.
|
||||
fault_key = uuid4().hex
|
||||
fault_text = "Atomic rollback " + fault_key
|
||||
sql(
|
||||
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
|
||||
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
|
||||
+ fault_key
|
||||
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
|
||||
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
|
||||
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
|
||||
)
|
||||
try:
|
||||
failed = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert failed.status_code == 500
|
||||
assert (
|
||||
sql(
|
||||
"SELECT count(*) FROM messages WHERE text='"
|
||||
+ fault_text
|
||||
+ "'"
|
||||
)
|
||||
== "0"
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
"DROP TRIGGER student_idempo_fault ON idempotency; "
|
||||
"DROP FUNCTION student_idempo_fault()"
|
||||
)
|
||||
retried = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert retried.status_code == 201
|
||||
report["atomic_rollback_and_5xx_retry"] = True
|
||||
sql(
|
||||
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE key='" + fault_key + "'"
|
||||
)
|
||||
expired = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": "New retention window"},
|
||||
)
|
||||
assert expired.status_code == 201
|
||||
assert expired.json()["id"] != retried.json()["id"]
|
||||
report["expired_key_can_be_reused"] = True
|
||||
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
|
||||
report["failovers"] = []
|
||||
for api in apis:
|
||||
|
||||
def stop(target=api["instance"]):
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
||||
subprocess.run(
|
||||
["scripts/stop-one.sh"],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Failover", "member_ids": [user["id"]]},
|
||||
)
|
||||
result = exercise(client, session, r.json(), stop)
|
||||
assert api["instance"] not in [
|
||||
o["instance"] for o in result["observations"][-5:]
|
||||
]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.State.Running}}",
|
||||
api["container"],
|
||||
]
|
||||
)
|
||||
== "false"
|
||||
)
|
||||
result["stopped"] = api
|
||||
report["failovers"].append(result)
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Rolling", "member_ids": [user["id"]]},
|
||||
)
|
||||
before_roll = snapshot()
|
||||
report["rolling"] = exercise(
|
||||
client,
|
||||
session,
|
||||
r.json(),
|
||||
lambda: command(["scripts/rolling-update.sh"]),
|
||||
seconds=45,
|
||||
)
|
||||
after_roll = snapshot()
|
||||
assert not (
|
||||
{c["container"] for c in before_roll if c["role"] == "api"}
|
||||
& {c["container"] for c in after_roll if c["role"] == "api"}
|
||||
)
|
||||
report["rolling"]["before"] = before_roll
|
||||
report["rolling"]["after"] = after_roll
|
||||
replay = repeat(0)[0]
|
||||
assert replay == repeated[0][0]
|
||||
report["replay_after_replacement"] = True
|
||||
before = snapshot()
|
||||
env = {
|
||||
**os.environ,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
for mode, hook in [
|
||||
("failover", "scripts/stop-one.sh"),
|
||||
("persistence", "scripts/restart.sh"),
|
||||
]:
|
||||
if mode == "persistence":
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
before = snapshot()
|
||||
run = subprocess.run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
mode,
|
||||
"--action-script",
|
||||
hook,
|
||||
"--base-url",
|
||||
BASE,
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert run.returncode == 0, run.stdout + run.stderr
|
||||
report["public_" + mode] = run.stdout.strip()
|
||||
after = snapshot()
|
||||
for role in ["postgres", "cache", "s3", "broker", "lb"]:
|
||||
old = next(c for c in before if c["role"] == role)
|
||||
new = next(c for c in after if c["role"] == role)
|
||||
assert old["started_at"] != new["started_at"]
|
||||
report["after_persistence"] = after
|
||||
report["complete"] = True
|
||||
finally:
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: process failover, rolling, replay and full persistence")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,687 @@
|
||||
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
|
||||
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
|
||||
PREFIX = """import io,json,hashlib,time,urllib.request
|
||||
from uuid import UUID,uuid4
|
||||
from psycopg.types.json import Jsonb
|
||||
from src.pipeline import database,broker,envelope,STREAM,GROUP
|
||||
from src.object_store import ObjectStore,BUCKET
|
||||
from src.operator import change
|
||||
from src.reconcile import reconcile
|
||||
db,store,queue=database(),ObjectStore(),broker()
|
||||
"""
|
||||
|
||||
|
||||
def inside(code):
|
||||
cid = compose("ps", "-q", "controller").split()[0]
|
||||
return json.loads(
|
||||
command(
|
||||
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def wait(fn, timeout=90):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
value = fn()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.15)
|
||||
raise TimeoutError("Fault scenario condition not reached")
|
||||
|
||||
|
||||
def stop(role):
|
||||
compose("stop", "--timeout", "1", role)
|
||||
|
||||
|
||||
def start(role):
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", role)
|
||||
|
||||
|
||||
def row(job):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
|
||||
|
||||
def inject(job, phase, action, remaining=1):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
|
||||
)
|
||||
|
||||
|
||||
def hit(identity):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
|
||||
)
|
||||
|
||||
|
||||
def operator(job, version=None):
|
||||
return inside(
|
||||
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
|
||||
)
|
||||
|
||||
|
||||
def duplicate(job, count=5):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
|
||||
)
|
||||
|
||||
|
||||
def inspect_effect(job, *, pending=False):
|
||||
return inside(f"""with db.connection() as c:
|
||||
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
|
||||
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
|
||||
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
|
||||
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
|
||||
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
|
||||
pending_entries=[]
|
||||
if {pending!r}:
|
||||
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
|
||||
entry=item['message_id']
|
||||
messages=queue.xrange(STREAM,min=entry,max=entry)
|
||||
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
|
||||
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
|
||||
parser.add_argument("--repeats", type=int, default=2)
|
||||
parser.add_argument(
|
||||
"--start-at", choices=["all", "versions"], default="all"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"track": 2,
|
||||
"scope": args.start_at,
|
||||
"complete": False,
|
||||
"cases": [],
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save(name, details):
|
||||
report["cases"].append({"case": name, **details})
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS:", name, flush=True)
|
||||
|
||||
try:
|
||||
ready()
|
||||
# Check instrumentation is really enabled and production default is off.
|
||||
cid = compose("ps", "-q", "api").split()[0]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import os;print(os.getenv('PIPELINE_FAULTS'))",
|
||||
]
|
||||
)
|
||||
== "1"
|
||||
)
|
||||
stop("reconciler")
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=25
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
h = auth(session)
|
||||
c = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=h,
|
||||
json={
|
||||
"title": "Pipeline synthetic faults",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert c.status_code == 201
|
||||
chat = c.json()["id"]
|
||||
raw_io = io.BytesIO()
|
||||
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
|
||||
raw_io, format="PNG"
|
||||
)
|
||||
raw = raw_io.getvalue()
|
||||
|
||||
def upload():
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**h, "Content-Type": "image/png"},
|
||||
params={"filename": "synthetic.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202, r.text
|
||||
a = r.json()
|
||||
assert a["status"] == "queued" and not a["variants"]
|
||||
j = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
return a, j
|
||||
|
||||
def metadata(a):
|
||||
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
def until(a, state="ready", timeout=90):
|
||||
return wait(
|
||||
lambda: (
|
||||
r if (r := metadata(a))["status"] == state else None
|
||||
),
|
||||
timeout,
|
||||
)
|
||||
|
||||
def confirm(a, j):
|
||||
until(a)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.status_code == 200 and downloaded.content == raw
|
||||
)
|
||||
m = metadata(a)
|
||||
assert (
|
||||
m["sha256"] == hashlib.sha256(raw).hexdigest()
|
||||
and m["error_code"] is None
|
||||
)
|
||||
assert {v["name"] for v in m["variants"]} == {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}
|
||||
for v in m["variants"]:
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=h,
|
||||
params={"variant": v["name"]},
|
||||
)
|
||||
assert (
|
||||
r.status_code == 200
|
||||
and hashlib.sha256(r.content).hexdigest()
|
||||
== v["sha256"]
|
||||
)
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert (
|
||||
image.size
|
||||
== {
|
||||
"thumbnail": (256, 160),
|
||||
"crop": (128, 128),
|
||||
"optimized": (320, 200),
|
||||
}[v["name"]]
|
||||
)
|
||||
assert not image.getexif()
|
||||
effect = inspect_effect(j)
|
||||
assert (
|
||||
effect["publications"] == 1
|
||||
and len(effect["variants"]) == 4
|
||||
and len(effect["physical_keys"]) == 4
|
||||
)
|
||||
return effect
|
||||
|
||||
if args.start_at == "all":
|
||||
# Source-only crash before the DB handoff must never be reported as accepted.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
marker = inject(chat, "api_after_staging", "wait")
|
||||
before = inside(
|
||||
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
||||
)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(
|
||||
lambda: client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**h, "Content-Type": "image/png"},
|
||||
params={"filename": "unconfirmed.png"},
|
||||
content=raw,
|
||||
)
|
||||
)
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
compose("kill", "-s", "SIGKILL", "api")
|
||||
try:
|
||||
response = future.result(15)
|
||||
assert response.status_code != 202
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
pool.shutdown()
|
||||
start("api")
|
||||
ready()
|
||||
after = inside(
|
||||
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
||||
)
|
||||
assert before == after
|
||||
save(
|
||||
"staging_before_handoff_crash",
|
||||
{
|
||||
"no_202": True,
|
||||
"no_attachment_or_job_created": True,
|
||||
"orphan_retained_for_cleanup": True,
|
||||
},
|
||||
)
|
||||
# API durable commit/outbox survives restart without a broker or worker.
|
||||
stop("broker")
|
||||
a, j = upload()
|
||||
linked = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**h, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
)
|
||||
assert linked.status_code == 201
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
compose("restart", "api")
|
||||
ready()
|
||||
start("broker")
|
||||
start("relay")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
save(
|
||||
"commit_before_publish_and_worker_stopped",
|
||||
{
|
||||
"attachment": a["id"],
|
||||
"effect": effect,
|
||||
"queued_link_allowed": True,
|
||||
"api_and_broker_restarted": True,
|
||||
"broker_stopped_during_upload": True,
|
||||
},
|
||||
)
|
||||
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
|
||||
)
|
||||
)
|
||||
stop("relay")
|
||||
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
|
||||
compose("kill", "-s", "SIGKILL", "broker")
|
||||
start("broker")
|
||||
wait(lambda: inside("print(json.dumps(queue.ping()))"))
|
||||
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
|
||||
assert restored == ids, {"before": ids, "after": restored}
|
||||
start("relay")
|
||||
start("worker")
|
||||
save(
|
||||
"confirmed_broker_entry_survives_sigkill",
|
||||
{
|
||||
"effect": confirm(a, j),
|
||||
"same_stream_entries": True,
|
||||
"entry_count": len(ids),
|
||||
},
|
||||
)
|
||||
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
a, j = upload()
|
||||
marker = inject(j, "relay_after_publish", "crash")
|
||||
start("relay")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
time.sleep(0.5)
|
||||
unpublished = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
|
||||
)
|
||||
assert unpublished >= 1
|
||||
start("relay")
|
||||
start("worker")
|
||||
save(
|
||||
"relay_crash_after_xadd_before_pg_marker",
|
||||
{
|
||||
"unpublished_after_crash": unpublished,
|
||||
"effect": confirm(a, j),
|
||||
},
|
||||
)
|
||||
# Repeated before/after effect and commit/ack windows.
|
||||
for repeat in range(args.repeats):
|
||||
for phase in [
|
||||
"before_effect",
|
||||
"after_effect",
|
||||
"after_commit",
|
||||
]:
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, phase, "wait")
|
||||
start("worker")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
at_crash = inspect_effect(
|
||||
j, pending=phase == "after_commit"
|
||||
)
|
||||
assert at_crash["job"]["state"] == (
|
||||
"ready"
|
||||
if phase == "after_commit"
|
||||
else "processing"
|
||||
)
|
||||
assert at_crash["publications"] == (
|
||||
1 if phase == "after_commit" else 0
|
||||
)
|
||||
assert len(at_crash["physical_keys"]) == (
|
||||
0 if phase == "before_effect" else 4
|
||||
)
|
||||
if phase == "after_commit":
|
||||
assert at_crash["pending_stream_entries"], (
|
||||
"Result must be committed while notification is still unacknowledged"
|
||||
)
|
||||
assert not any(
|
||||
e["event"] == "child_finished"
|
||||
for e in at_crash["events"]
|
||||
)
|
||||
compose("kill", "-s", "SIGKILL", "worker")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
if phase == "after_commit":
|
||||
wait(
|
||||
lambda j=j: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
and e["detail"]["state"] == "ready"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
wait(
|
||||
lambda j=j: inside(
|
||||
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
|
||||
)
|
||||
)
|
||||
effect = inspect_effect(j)
|
||||
else:
|
||||
assert effect["job"]["attempt"] >= 2
|
||||
save(
|
||||
f"{phase}_sigkill_repeat_{repeat + 1}",
|
||||
{"at_crash": at_crash, "effect": effect},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "transient")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
assert effect["job"]["attempt"] == 2
|
||||
save("transient_retry_with_jitter", {"effect": effect})
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, "before_effect", "hang")
|
||||
start("worker")
|
||||
wait(lambda: hit(marker))
|
||||
limits = inside(
|
||||
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
|
||||
)
|
||||
assert limits["address_space_bytes"] == [805306368, 805306368]
|
||||
assert limits["memory.max"] == "1610612736"
|
||||
effect = confirm(a, j)
|
||||
assert any(
|
||||
e["event"] == "retry"
|
||||
and e["detail"]["code"] == "worker_timeout"
|
||||
for e in effect["events"]
|
||||
)
|
||||
save(
|
||||
"hung_child_killed_and_retried",
|
||||
{"effect": effect, "observed_limits": limits},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "permanent")
|
||||
start("worker")
|
||||
failed = until(a, "failed")
|
||||
assert failed["error_code"] == "invalid_image"
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**h, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert operator(j) == j
|
||||
effect = confirm(a, j)
|
||||
assert effect["job"]["cycle"] == 2
|
||||
try:
|
||||
operator(j)
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("Replay of ready must be rejected")
|
||||
save(
|
||||
"permanent_failed_registry_and_audited_replay",
|
||||
{
|
||||
"failed": failed,
|
||||
"effect": effect,
|
||||
"ready_replay_rejected": True,
|
||||
},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "transient", remaining=5)
|
||||
start("worker")
|
||||
failed = until(a, "failed", timeout=150)
|
||||
assert (
|
||||
failed["error_code"] == "retry_exhausted"
|
||||
and row(j)["attempt"] == 5
|
||||
)
|
||||
operator(j)
|
||||
save(
|
||||
"finite_retry_exhaustion_then_explicit_replay",
|
||||
{"failed": failed, "effect": confirm(a, j)},
|
||||
)
|
||||
# Reverse notification order and add concurrent copies: one publication/key.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
a, j = upload()
|
||||
b, k = upload()
|
||||
duplicate(k, 8)
|
||||
duplicate(j, 8)
|
||||
start("worker")
|
||||
start("relay")
|
||||
ea = confirm(a, j)
|
||||
confirm(b, k)
|
||||
duplicate(j, 10)
|
||||
duplicate(k, 10)
|
||||
wait(
|
||||
lambda: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
assert (
|
||||
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
|
||||
)
|
||||
save(
|
||||
"reordered_and_duplicate_notifications",
|
||||
{
|
||||
"effects": [inspect_effect(j), inspect_effect(k)],
|
||||
"added_duplicates": 36,
|
||||
"no_extra_attempt_after_ready": True,
|
||||
},
|
||||
)
|
||||
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
|
||||
for repeat in range(args.repeats):
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, "after_effect", "wait")
|
||||
# Keep one controller process hot throughout the narrow stale-writer window.
|
||||
code = f"""from src.pipeline import event
|
||||
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
|
||||
end=time.monotonic()+45
|
||||
while True:
|
||||
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
|
||||
if hit: break
|
||||
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
|
||||
time.sleep(.05)
|
||||
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
|
||||
end=time.monotonic()+10
|
||||
while True:
|
||||
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
|
||||
if ready: break
|
||||
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
|
||||
time.sleep(.05)
|
||||
with db.connection() as c:
|
||||
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
|
||||
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
|
||||
end=time.monotonic()+10
|
||||
while True:
|
||||
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
|
||||
if stale: break
|
||||
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
|
||||
time.sleep(.05)
|
||||
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
|
||||
"""
|
||||
with ThreadPoolExecutor(max_workers=1) as hot:
|
||||
future = hot.submit(inside, code)
|
||||
wait(
|
||||
lambda j=j: inside(
|
||||
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
|
||||
)
|
||||
)
|
||||
start("worker")
|
||||
race = future.result(timeout=45)
|
||||
k = race["job_id"]
|
||||
hashes = race["hashes"]
|
||||
confirm(a, k)
|
||||
duplicate(j, 12)
|
||||
wait(
|
||||
lambda j=j: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
assert hashes == {
|
||||
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
|
||||
}
|
||||
assert inspect_effect(j)["publications"] == 0
|
||||
save(
|
||||
f"stale_v1_after_v2_repeat_{repeat + 1}",
|
||||
{
|
||||
"v1": inspect_effect(j),
|
||||
"v2": inspect_effect(k),
|
||||
"new_hashes_unchanged": True,
|
||||
},
|
||||
)
|
||||
# No arbitrary URL/bytes fields in queue envelopes.
|
||||
invalid = inside(
|
||||
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
|
||||
)
|
||||
assert invalid
|
||||
wait(
|
||||
lambda: inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
|
||||
)
|
||||
)
|
||||
save(
|
||||
"malformed_notification_rejected", {"durable_diagnostic": True}
|
||||
)
|
||||
# Private bucket, bounded IAM, and download after uploader process dies.
|
||||
a, j = upload()
|
||||
confirm(a, j)
|
||||
key = f"objects/{a['id']}/v1/original"
|
||||
security = inside(f"""from minio.error import S3Error
|
||||
from urllib.error import HTTPError
|
||||
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
|
||||
except HTTPError as e: anonymous=e.code
|
||||
else: raise AssertionError('Anonymous access allowed')
|
||||
denials={{}}
|
||||
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
|
||||
try: fn()
|
||||
except S3Error as e: denials[name]=e.code
|
||||
else: raise AssertionError('IAM allowed forbidden action')
|
||||
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
|
||||
assert security["anonymous"] == 403 and set(
|
||||
security["iam"].values()
|
||||
) == {"AccessDenied"}
|
||||
selected = None
|
||||
for cid in compose("ps", "-q", "api").split():
|
||||
alias = command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
||||
]
|
||||
)
|
||||
uploader = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
|
||||
)
|
||||
if alias == uploader:
|
||||
selected = cid
|
||||
assert selected
|
||||
command(["docker", "stop", "--time", "1", selected])
|
||||
ready()
|
||||
wait(
|
||||
lambda: (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 200
|
||||
)
|
||||
)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.content == raw
|
||||
and downloaded.headers["X-Instance-Id"] != uploader
|
||||
)
|
||||
start("api")
|
||||
ready()
|
||||
save(
|
||||
"private_iam_and_other_api_download",
|
||||
{
|
||||
"security": security,
|
||||
"attachment": a["id"],
|
||||
"uploader": uploader,
|
||||
"downloader": downloaded.headers["X-Instance-Id"],
|
||||
"sha256": hashlib.sha256(raw).hexdigest(),
|
||||
},
|
||||
)
|
||||
report["worker_metrics"] = inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
report["audit"] = inside(
|
||||
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
report["complete"] = True
|
||||
finally:
|
||||
for role in ["api", "broker", "relay", "worker", "reconciler"]:
|
||||
try:
|
||||
start(role)
|
||||
except Exception:
|
||||
pass
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
|
||||
def early_stop(signum, _frame):
|
||||
print(
|
||||
json.dumps({"event": "stopped_before_startup", "signal": signum}),
|
||||
flush=True,
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
signal.signal(signal.SIGTERM, early_stop)
|
||||
signal.signal(signal.SIGINT, early_stop)
|
||||
|
||||
import uvicorn # noqa: E402
|
||||
|
||||
from src.logging_config import JsonFormatter # noqa: E402
|
||||
|
||||
|
||||
def main() -> None:
|
||||
host = os.environ.get("APP_HOST", "127.0.0.1")
|
||||
port = int(os.environ.get("APP_PORT", "8080"))
|
||||
if not 1 <= port <= 65535:
|
||||
raise ValueError("APP_PORT must be between 1 and 65535")
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
|
||||
uvicorn.run(
|
||||
"src.api:app",
|
||||
host=host,
|
||||
port=port,
|
||||
workers=1,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=5,
|
||||
log_config=None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+424
@@ -0,0 +1,424 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from anyio import Semaphore, fail_after, to_thread
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from pydantic import ValidationError
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from src import images
|
||||
from src.attachments import Attachments
|
||||
from src.cache import UserCache
|
||||
from src.cursor import parse_uuid
|
||||
from src.database import Database
|
||||
from src.domain import timestamp
|
||||
from src.errors import DomainError, invalid
|
||||
from src.object_store import ObjectStore
|
||||
from src.password import Passwords
|
||||
from src.repository import PostgresRepository
|
||||
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
|
||||
from src.service import MessengerService
|
||||
from src.settings import Settings
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
INSTANCE_ID = uuid4().hex
|
||||
|
||||
|
||||
def response(data: object, *, status: int = 200) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
jsonable_encoder(
|
||||
data, custom_encoder={datetime: timestamp, UUID: str}
|
||||
),
|
||||
status_code=status,
|
||||
)
|
||||
|
||||
|
||||
async def body(request: Request, model: type[Input]) -> Input:
|
||||
try:
|
||||
data = await request.body()
|
||||
return model.model_validate(
|
||||
json.loads(
|
||||
data.decode("utf-8"),
|
||||
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
|
||||
)
|
||||
)
|
||||
except ValueError, ValidationError, UnicodeError, RecursionError:
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
def pagination(request: Request) -> tuple[int, str | None]:
|
||||
limits = request.query_params.getlist("limit")
|
||||
cursors = request.query_params.getlist("cursor")
|
||||
if len(limits) > 1 or len(cursors) > 1:
|
||||
raise invalid()
|
||||
raw = limits[0] if limits else "50"
|
||||
if not raw.isascii() or not raw.isdecimal():
|
||||
raise invalid()
|
||||
raw = raw.lstrip("0") or "0"
|
||||
if len(raw) > 3:
|
||||
raise invalid()
|
||||
limit = int(raw)
|
||||
if not 1 <= limit <= 100:
|
||||
raise invalid()
|
||||
return limit, cursors[0] if cursors else None
|
||||
|
||||
|
||||
def service(request: Request) -> MessengerService:
|
||||
return request.app.state.service
|
||||
|
||||
|
||||
async def run_io(function, **kwargs):
|
||||
return await to_thread.run_sync(partial(function, **kwargs))
|
||||
|
||||
|
||||
async def actor(request: Request):
|
||||
return await run_io(
|
||||
service(request).identify, value=request.headers.get("authorization")
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
settings = Settings.load()
|
||||
database = Database(url=settings.database_url, size=settings.pool_size)
|
||||
cache = UserCache(url=settings.cache_url)
|
||||
store = ObjectStore()
|
||||
try:
|
||||
await run_io(database.open)
|
||||
passwords = await run_io(Passwords)
|
||||
app.state.database = database
|
||||
app.state.store = store
|
||||
app.state.upload_slots = Semaphore(2)
|
||||
app.state.attachments = Attachments(
|
||||
database=database, store=store, instance=INSTANCE_ID
|
||||
)
|
||||
app.state.service = MessengerService(
|
||||
repository=PostgresRepository(database=database, cache=cache),
|
||||
passwords=passwords,
|
||||
session_ttl=settings.session_ttl,
|
||||
)
|
||||
app.state.ready = True
|
||||
yield
|
||||
finally:
|
||||
app.state.ready = False
|
||||
await run_io(store.close)
|
||||
await run_io(cache.close)
|
||||
await run_io(database.close)
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(request: Request, call_next):
|
||||
request_id = str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
result = await call_next(request)
|
||||
except Exception:
|
||||
logger.error(
|
||||
json.dumps({"event": "internal_error", "request_id": request_id})
|
||||
)
|
||||
result = response(
|
||||
{
|
||||
"error": {
|
||||
"code": "internal_error",
|
||||
"message": "Internal error",
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
result.headers["X-Request-Id"] = request_id
|
||||
result.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
route = request.scope.get("route")
|
||||
logger.info(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "request",
|
||||
"request_id": request_id,
|
||||
"method": request.method,
|
||||
"route": getattr(route, "path", "unmatched"),
|
||||
"status": result.status_code,
|
||||
"elapsed_ms": round((time.perf_counter() - started) * 1000, 3),
|
||||
}
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@app.exception_handler(DomainError)
|
||||
async def domain_error(request: Request, error: DomainError):
|
||||
result = response(
|
||||
{
|
||||
"error": {
|
||||
"code": error.code,
|
||||
"message": error.message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status=error.status,
|
||||
)
|
||||
result.headers.update(error.headers)
|
||||
return result
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, error: HTTPException):
|
||||
code = "not_found" if error.status_code == 404 else "invalid_request"
|
||||
return response(
|
||||
{
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": "Resource not found"
|
||||
if error.status_code == 404
|
||||
else "Invalid request",
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status=error.status_code,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
async def live():
|
||||
return response({"status": "ok"})
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready():
|
||||
if not getattr(app.state, "ready", False):
|
||||
raise DomainError(503, "unavailable", "Not ready")
|
||||
await run_io(app.state.database.ping)
|
||||
await run_io(app.state.store.ping)
|
||||
return response({"status": "ok"})
|
||||
|
||||
|
||||
@app.post("/api/v1/users")
|
||||
async def register(request: Request):
|
||||
data = await body(request, CreateUser)
|
||||
return response(
|
||||
await run_io(service(request).register, **data.model_dump()),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def current(request: Request):
|
||||
return response(await actor(request))
|
||||
|
||||
|
||||
@app.post("/api/v1/chats")
|
||||
async def create_chat(request: Request):
|
||||
user = await actor(request)
|
||||
data = await body(request, CreateChat)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).create_chat, actor_id=user.id, **data.model_dump()
|
||||
),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(request: Request):
|
||||
user = await actor(request)
|
||||
limit, cursor = pagination(request)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).chats,
|
||||
actor_id=user.id,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).get_chat,
|
||||
actor_id=user.id,
|
||||
chat_id=parse_uuid(chat_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages")
|
||||
async def send(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
|
||||
data = await body(request, CreateMessage)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).send,
|
||||
actor_id=user.id,
|
||||
chat_id=cid,
|
||||
idempotency_key=request.headers.get("idempotency-key"),
|
||||
**data.model_dump(),
|
||||
),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def history(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
chat = await run_io(
|
||||
service(request).get_chat, actor_id=user.id, chat_id=cid
|
||||
)
|
||||
limit, cursor = pagination(request)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).history,
|
||||
actor_id=user.id,
|
||||
chat=chat,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions")
|
||||
async def login(request: Request):
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).login,
|
||||
authorization=request.headers.get("authorization"),
|
||||
),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current")
|
||||
async def logout(request: Request):
|
||||
await run_io(
|
||||
service(request).logout,
|
||||
authorization=request.headers.get("authorization"),
|
||||
)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments")
|
||||
async def upload(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
|
||||
names = request.query_params.getlist("filename")
|
||||
if len(names) != 1:
|
||||
raise invalid()
|
||||
name = images.filename(names[0])
|
||||
declared = (
|
||||
request.headers.get("content-type", "")
|
||||
.partition(";")[0]
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
if declared not in {"image/png", "image/jpeg"}:
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "PNG or JPEG required"
|
||||
)
|
||||
length = request.headers.get("content-length")
|
||||
if length and (not length.isdecimal() or len(length) > 12):
|
||||
raise invalid()
|
||||
if length and int(length) > images.MAX_BYTES:
|
||||
raise DomainError(413, "payload_too_large", "Byte limit exceeded")
|
||||
try:
|
||||
with fail_after(20):
|
||||
async with app.state.upload_slots:
|
||||
with tempfile.SpooledTemporaryFile(
|
||||
max_size=1024 * 1024, mode="w+b"
|
||||
) as stream:
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > images.MAX_BYTES:
|
||||
raise DomainError(
|
||||
413, "payload_too_large", "Byte limit exceeded"
|
||||
)
|
||||
digest.update(chunk)
|
||||
stream.write(chunk)
|
||||
await run_io(
|
||||
images.verify, stream=stream, declared=declared
|
||||
)
|
||||
result = await run_io(
|
||||
app.state.attachments.upload,
|
||||
chat_id=cid,
|
||||
owner_id=user.id,
|
||||
filename=name,
|
||||
content_type=declared,
|
||||
size=size,
|
||||
sha256=digest.hexdigest(),
|
||||
stream=stream,
|
||||
)
|
||||
return response(result, status=202)
|
||||
except ClientDisconnect:
|
||||
raise invalid() from None
|
||||
except TimeoutError:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Upload deadline exceeded"
|
||||
) from None
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def attachment_metadata(request: Request, attachment_id: str):
|
||||
user = await actor(request)
|
||||
return response(
|
||||
await run_io(
|
||||
app.state.attachments.metadata,
|
||||
identity=parse_uuid(attachment_id),
|
||||
actor_id=user.id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def attachment_content(request: Request, attachment_id: str):
|
||||
user = await actor(request)
|
||||
variants = request.query_params.getlist("variant")
|
||||
if len(variants) > 1:
|
||||
raise invalid()
|
||||
row, upstream = await run_io(
|
||||
app.state.attachments.content,
|
||||
identity=parse_uuid(attachment_id),
|
||||
actor_id=user.id,
|
||||
variant=variants[0] if variants else "original",
|
||||
)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := upstream.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
upstream.close()
|
||||
upstream.release_conn()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,144 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from src.errors import DomainError, not_found
|
||||
from src.pipeline import enqueue, fault
|
||||
|
||||
|
||||
def dto(row, variants=()):
|
||||
return {
|
||||
k: row[k]
|
||||
for k in [
|
||||
"id",
|
||||
"chat_id",
|
||||
"owner_id",
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
} | {
|
||||
"status": row["state"],
|
||||
"variants": [
|
||||
{
|
||||
k: v[k]
|
||||
for k in [
|
||||
"name",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"width",
|
||||
"height",
|
||||
]
|
||||
}
|
||||
for v in variants
|
||||
if v["name"] != "original"
|
||||
],
|
||||
"error_code": row["error_code"] if row["state"] == "failed" else None,
|
||||
}
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, *, database, store, instance):
|
||||
self.database, self.store, self.instance = database, store, instance
|
||||
|
||||
def upload(
|
||||
self,
|
||||
*,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size,
|
||||
sha256,
|
||||
stream,
|
||||
):
|
||||
identity, job_id = uuid4(), uuid4()
|
||||
key = f"objects/staging/{identity}/original"
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)",
|
||||
(identity, key),
|
||||
)
|
||||
self.store.put(
|
||||
key=key,
|
||||
stream=stream,
|
||||
size=size,
|
||||
content_type=content_type,
|
||||
sha256=sha256,
|
||||
)
|
||||
fault(self.database, chat_id, "api_after_staging")
|
||||
with self.database.connection() as conn, conn.transaction():
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE",
|
||||
(identity,),
|
||||
).fetchone()
|
||||
if reservation is None:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Upload reservation expired"
|
||||
)
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) "
|
||||
"VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *",
|
||||
(
|
||||
identity,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size,
|
||||
sha256,
|
||||
key,
|
||||
self.instance,
|
||||
),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')",
|
||||
(job_id, identity, key),
|
||||
)
|
||||
enqueue(conn, job_id)
|
||||
conn.execute(
|
||||
"DELETE FROM upload_reservations WHERE id=%s", (identity,)
|
||||
)
|
||||
fault(self.database, chat_id, "api_after_commit")
|
||||
# Captured committed snapshot remains queued even if worker already finished.
|
||||
return dto(row)
|
||||
|
||||
def row(self, *, identity, actor_id):
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s",
|
||||
(actor_id, identity),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
def metadata(self, *, identity, actor_id):
|
||||
row = self.row(identity=identity, actor_id=actor_id)
|
||||
variants = []
|
||||
if row["state"] == "ready":
|
||||
with self.database.connection() as conn:
|
||||
variants = conn.execute(
|
||||
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
|
||||
(identity, row["published_version"]),
|
||||
).fetchall()
|
||||
return dto(row, variants)
|
||||
|
||||
def content(self, *, identity, actor_id, variant):
|
||||
row = self.row(identity=identity, actor_id=actor_id)
|
||||
if row["state"] != "ready":
|
||||
raise DomainError(409, "conflict", "Attachment is not ready")
|
||||
with self.database.connection() as conn:
|
||||
data = conn.execute(
|
||||
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
|
||||
(identity, row["published_version"], variant),
|
||||
).fetchone()
|
||||
if data is None:
|
||||
raise not_found()
|
||||
if self.store.head(key=data["object_key"]) is None:
|
||||
raise DomainError(503, "unavailable", "Variant unavailable")
|
||||
return {**data, "filename": row["filename"]}, self.store.get(
|
||||
key=data["object_key"]
|
||||
)
|
||||
@@ -0,0 +1,66 @@
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
from src.domain import User, timestamp
|
||||
|
||||
|
||||
class UserCache:
|
||||
def __init__(self, *, url: str):
|
||||
self.client = redis.Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=0.15,
|
||||
socket_timeout=0.15,
|
||||
max_connections=16,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
decode_responses=True,
|
||||
)
|
||||
self.disabled_until = 0.0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
|
||||
def _command(self, function, *args, **kwargs):
|
||||
if time.monotonic() < self.disabled_until:
|
||||
return None
|
||||
try:
|
||||
return function(*args, **kwargs)
|
||||
except redis.RedisError, OSError:
|
||||
with self.lock:
|
||||
self.disabled_until = time.monotonic() + 5
|
||||
return None
|
||||
|
||||
def get(self, *, user_id: UUID) -> User | None:
|
||||
raw = self._command(self.client.get, "user:" + str(user_id))
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
row = json.loads(raw)
|
||||
if UUID(row["id"]) != user_id:
|
||||
return None
|
||||
return User(
|
||||
user_id,
|
||||
row["username"],
|
||||
row["display_name"],
|
||||
datetime.fromisoformat(row["created_at"]),
|
||||
)
|
||||
except ValueError, TypeError, KeyError:
|
||||
return None
|
||||
|
||||
def put(self, *, user: User):
|
||||
raw = json.dumps(
|
||||
{
|
||||
"id": str(user.id),
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"created_at": timestamp(user.created_at),
|
||||
}
|
||||
)
|
||||
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
|
||||
@@ -0,0 +1,64 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from src.domain import Key, timestamp
|
||||
from src.errors import invalid
|
||||
|
||||
|
||||
def parse_uuid(value: str) -> UUID:
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
|
||||
):
|
||||
raise invalid()
|
||||
return UUID(value)
|
||||
|
||||
|
||||
def encode_cursor(*, scope: str, key: Key) -> str:
|
||||
payload = {
|
||||
"v": 1,
|
||||
"scope": scope,
|
||||
"at": timestamp(key[0]),
|
||||
"id": str(key[1]),
|
||||
}
|
||||
return (
|
||||
base64.urlsafe_b64encode(
|
||||
json.dumps(payload, separators=(",", ":")).encode()
|
||||
)
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
|
||||
|
||||
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if (
|
||||
not value
|
||||
or len(value) > 2048
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
|
||||
):
|
||||
raise ValueError
|
||||
data = json.loads(
|
||||
base64.b64decode(
|
||||
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
|
||||
)
|
||||
)
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or type(data.get("v")) is not int
|
||||
or data["v"] != 1
|
||||
or data.get("scope") != scope
|
||||
or not isinstance(data.get("at"), str)
|
||||
):
|
||||
raise ValueError
|
||||
at = datetime.fromisoformat(data["at"])
|
||||
if at.tzinfo is None or timestamp(at) != data["at"]:
|
||||
raise ValueError
|
||||
return at, parse_uuid(data["id"])
|
||||
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
|
||||
raise invalid() from None
|
||||
@@ -0,0 +1,69 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
from src.errors import DomainError
|
||||
|
||||
|
||||
class DeadlineConnection(psycopg.Connection):
|
||||
"""Клиентский срок действует и при замороженном PostgreSQL."""
|
||||
|
||||
def wait(self, gen, interval=0.1, timeout=None):
|
||||
deadline = getattr(self, "request_deadline", None)
|
||||
remaining = (
|
||||
max(0.001, deadline - time.monotonic()) if deadline else 3.0
|
||||
)
|
||||
bound = min(remaining, timeout) if timeout is not None else remaining
|
||||
try:
|
||||
return super().wait(gen, interval=interval, timeout=bound)
|
||||
except psycopg.OperationalError:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, *, url: str, size: int):
|
||||
self.pool = ConnectionPool(
|
||||
url,
|
||||
connection_class=DeadlineConnection,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 2,
|
||||
"options": (
|
||||
"-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||
"-c idle_in_transaction_session_timeout=6000"
|
||||
),
|
||||
},
|
||||
min_size=1,
|
||||
max_size=size,
|
||||
timeout=1,
|
||||
open=False,
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=10)
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=2)
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with self.pool.connection(timeout=1) as conn:
|
||||
conn.request_deadline = time.monotonic() + 4
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.request_deadline = None
|
||||
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Database unavailable"
|
||||
) from None
|
||||
|
||||
def ping(self):
|
||||
with self.connection() as conn:
|
||||
conn.execute("SELECT 1").fetchone()
|
||||
@@ -0,0 +1,45 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def timestamp(value: datetime) -> str:
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: tuple[UUID, ...]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: tuple[UUID, ...] = ()
|
||||
|
||||
|
||||
type Record = User | Chat | Message
|
||||
type Key = tuple[datetime, UUID]
|
||||
|
||||
|
||||
def order_key(record: Record) -> Key:
|
||||
return record.created_at, record.id
|
||||
@@ -0,0 +1,16 @@
|
||||
class DomainError(Exception):
|
||||
def __init__(
|
||||
self, status: int, code: str, message: str, headers=None
|
||||
) -> None:
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid() -> DomainError:
|
||||
return DomainError(400, "invalid_request", "Invalid request")
|
||||
|
||||
|
||||
def not_found() -> DomainError:
|
||||
return DomainError(404, "not_found", "Resource not found")
|
||||
@@ -0,0 +1,77 @@
|
||||
import struct
|
||||
import threading
|
||||
import unicodedata
|
||||
import zlib
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from src.errors import DomainError, invalid
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
DECODE_SLOT = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
def filename(value):
|
||||
if (
|
||||
not value
|
||||
or not 1 <= len(value) <= 255
|
||||
or not value.strip()
|
||||
or any(
|
||||
c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"}
|
||||
for c in value
|
||||
)
|
||||
):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def verify(stream, *, declared):
|
||||
if declared not in {"image/png", "image/jpeg"}:
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "PNG or JPEG required"
|
||||
)
|
||||
with DECODE_SLOT:
|
||||
try:
|
||||
stream.seek(0)
|
||||
with Image.open(stream) as image:
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(
|
||||
image.format
|
||||
)
|
||||
if actual != declared:
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "Image type mismatch"
|
||||
)
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise DomainError(
|
||||
413, "payload_too_large", "Pixel limit exceeded"
|
||||
)
|
||||
image.verify()
|
||||
# JPEG verify alone does not decode entropy data.
|
||||
stream.seek(0)
|
||||
with Image.open(stream) as image:
|
||||
image.load()
|
||||
except Image.DecompressionBombError:
|
||||
raise DomainError(
|
||||
413, "payload_too_large", "Pixel limit exceeded"
|
||||
) from None
|
||||
except (
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
SyntaxError,
|
||||
struct.error,
|
||||
zlib.error,
|
||||
EOFError,
|
||||
):
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "Invalid image"
|
||||
) from None
|
||||
except MemoryError:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Image memory budget exceeded"
|
||||
) from None
|
||||
finally:
|
||||
stream.seek(0)
|
||||
return declared
|
||||
@@ -0,0 +1,17 @@
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
message = record.getMessage()
|
||||
try:
|
||||
event = json.loads(message)
|
||||
if not isinstance(event, dict):
|
||||
event = {"event": message}
|
||||
except ValueError:
|
||||
event = {"event": message}
|
||||
event["timestamp"] = datetime.now(UTC).isoformat()
|
||||
event["level"] = record.levelname.lower()
|
||||
return json.dumps(event, ensure_ascii=False)
|
||||
@@ -0,0 +1,51 @@
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from src.settings import secret
|
||||
|
||||
|
||||
def main():
|
||||
with psycopg.connect(
|
||||
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
|
||||
) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(2301002)")
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(name text PRIMARY KEY, checksum text NOT NULL, "
|
||||
"applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
for path in sorted(Path("migrations").glob("*.sql")):
|
||||
content = path.read_bytes()
|
||||
checksum = hashlib.sha256(content).hexdigest()
|
||||
row = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE name=%s",
|
||||
(path.name,),
|
||||
).fetchone()
|
||||
if row:
|
||||
if row[0] != checksum:
|
||||
raise RuntimeError(
|
||||
"Applied migration checksum mismatch"
|
||||
)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(content.decode())
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations(name,checksum) "
|
||||
"VALUES(%s,%s)",
|
||||
(path.name, checksum),
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{"event": "migration_applied", "name": path.name}
|
||||
)
|
||||
)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(2301002)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
from contextlib import contextmanager
|
||||
|
||||
import urllib3
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
|
||||
from src.errors import DomainError
|
||||
from src.settings import secret
|
||||
|
||||
BUCKET = "messenger"
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.http = urllib3.PoolManager(
|
||||
maxsize=32,
|
||||
block=True,
|
||||
timeout=urllib3.Timeout(connect=1, read=3),
|
||||
retries=urllib3.Retry(total=0),
|
||||
)
|
||||
self.client = Minio(
|
||||
"s3:9000",
|
||||
access_key=secret("S3_ACCESS_KEY"),
|
||||
secret_key=secret("S3_SECRET_KEY"),
|
||||
secure=False,
|
||||
region="us-east-1",
|
||||
http_client=self.http,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def operation(self):
|
||||
try:
|
||||
yield
|
||||
except S3Error, urllib3.exceptions.HTTPError, OSError:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Object store unavailable"
|
||||
) from None
|
||||
|
||||
def ping(self):
|
||||
with self.operation():
|
||||
if not self.client.bucket_exists(BUCKET):
|
||||
raise DomainError(
|
||||
503, "unavailable", "Object store unavailable"
|
||||
)
|
||||
|
||||
def put(self, *, key, stream, size, content_type, sha256):
|
||||
stream.seek(0)
|
||||
with self.operation():
|
||||
self.client.put_object(
|
||||
BUCKET,
|
||||
key,
|
||||
stream,
|
||||
length=size,
|
||||
content_type=content_type,
|
||||
metadata={"sha256": sha256},
|
||||
part_size=10 * 1024 * 1024,
|
||||
num_parallel_uploads=1,
|
||||
)
|
||||
|
||||
def head(self, *, key):
|
||||
with self.operation():
|
||||
try:
|
||||
return self.client.stat_object(BUCKET, key)
|
||||
except S3Error as exc:
|
||||
if exc.code in {"NoSuchKey", "NoSuchObject"}:
|
||||
return None
|
||||
raise
|
||||
|
||||
def get(self, *, key):
|
||||
with self.operation():
|
||||
return self.client.get_object(BUCKET, key)
|
||||
|
||||
def delete(self, *, key):
|
||||
with self.operation():
|
||||
self.client.remove_object(BUCKET, key)
|
||||
|
||||
def objects(self):
|
||||
with self.operation():
|
||||
yield from self.client.list_objects(
|
||||
BUCKET, prefix="objects/", recursive=True
|
||||
)
|
||||
|
||||
def close(self):
|
||||
self.http.clear()
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import database, enqueue, event
|
||||
|
||||
|
||||
def change(db, store, *, job_id, operator, reason, version=None):
|
||||
if (
|
||||
not operator.strip()
|
||||
or not reason.strip()
|
||||
or len(operator) > 100
|
||||
or len(reason) > 500
|
||||
):
|
||||
raise ValueError("Operator and reason are required")
|
||||
with db.connection() as conn, conn.transaction():
|
||||
job = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
if not job:
|
||||
raise ValueError("Unknown job")
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
|
||||
(job["attachment_id"],),
|
||||
).fetchone()
|
||||
if job["pipeline_version"] != attachment["pipeline_version"]:
|
||||
raise ValueError("Superseded job cannot be replayed")
|
||||
source = (
|
||||
job["source_key"]
|
||||
if attachment["published_version"] is None
|
||||
else attachment["object_key"]
|
||||
)
|
||||
if store.head(key=source) is None:
|
||||
raise ValueError("Source expired or missing; replay refused")
|
||||
if version is None:
|
||||
if job["state"] != "failed":
|
||||
raise ValueError("Only failed jobs can be replayed")
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
action = "replay"
|
||||
target = job_id
|
||||
else:
|
||||
if (
|
||||
version not in {1, 2}
|
||||
or version <= attachment["pipeline_version"]
|
||||
):
|
||||
raise ValueError("Supported version must increase (1 or 2)")
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
target = uuid4()
|
||||
conn.execute(
|
||||
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
|
||||
(target, attachment["id"], version, source),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
|
||||
(version, attachment["id"]),
|
||||
)
|
||||
action = "new_version"
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
|
||||
(target, action, operator, reason),
|
||||
)
|
||||
event(conn, target, action, {"previous_job": str(job_id)})
|
||||
enqueue(conn, target)
|
||||
return str(target)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["failed", "replay", "version"])
|
||||
p.add_argument("--job-id", type=UUID)
|
||||
p.add_argument("--operator")
|
||||
p.add_argument("--reason")
|
||||
p.add_argument("--version", type=int)
|
||||
a = p.parse_args()
|
||||
db, store = database(), ObjectStore()
|
||||
try:
|
||||
if a.action == "failed":
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=str))
|
||||
else:
|
||||
if not all([a.job_id, a.operator, a.reason]):
|
||||
p.error("job-id/operator/reason required")
|
||||
if a.action == "version" and a.version is None:
|
||||
p.error("version required")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"job_id": change(
|
||||
db,
|
||||
store,
|
||||
job_id=a.job_id,
|
||||
operator=a.operator,
|
||||
reason=a.reason,
|
||||
version=a.version
|
||||
if a.action == "version"
|
||||
else None,
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
import threading
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from argon2.low_level import Type
|
||||
|
||||
from src.errors import DomainError
|
||||
|
||||
|
||||
class Passwords:
|
||||
def __init__(self):
|
||||
self.hasher = PasswordHasher(
|
||||
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
|
||||
)
|
||||
self.slots = threading.BoundedSemaphore(2)
|
||||
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
|
||||
|
||||
def _run(self, function, *args):
|
||||
if not self.slots.acquire(timeout=5):
|
||||
raise DomainError(503, "unavailable", "Authentication busy")
|
||||
try:
|
||||
return function(*args)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def hash(self, *, password: str) -> str:
|
||||
return self._run(self.hasher.hash, password)
|
||||
|
||||
def verify(self, *, password: str, encoded: str | None) -> bool:
|
||||
try:
|
||||
return (
|
||||
self._run(self.hasher.verify, encoded or self.dummy, password)
|
||||
and encoded is not None
|
||||
)
|
||||
except VerificationError, InvalidHashError:
|
||||
return False
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
"""Durable job state is authoritative; stream entries are wake-up notifications."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from src.database import Database
|
||||
from src.settings import integer, secret
|
||||
|
||||
STREAM = "attachments"
|
||||
GROUP = "image-workers"
|
||||
|
||||
|
||||
def database():
|
||||
db = Database(url=secret("DATABASE_URL"), size=4)
|
||||
db.open()
|
||||
return db
|
||||
|
||||
|
||||
def broker():
|
||||
return redis.Redis.from_url(
|
||||
secret("BROKER_URL"),
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=3,
|
||||
max_connections=8,
|
||||
)
|
||||
|
||||
|
||||
def envelope(row):
|
||||
return dict(
|
||||
schema_version=1,
|
||||
job_id=str(row["id"]),
|
||||
type="attachment.process.v1",
|
||||
attachment_id=str(row["attachment_id"]),
|
||||
pipeline_version=row["pipeline_version"],
|
||||
attempt=row["attempt"] + 1,
|
||||
created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
)
|
||||
|
||||
|
||||
def validate_job(data):
|
||||
required = {
|
||||
"schema_version",
|
||||
"job_id",
|
||||
"type",
|
||||
"attachment_id",
|
||||
"pipeline_version",
|
||||
"attempt",
|
||||
"created_at",
|
||||
}
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or not required <= data.keys()
|
||||
or data.keys() - required - {"traceparent", "tracestate"}
|
||||
):
|
||||
raise ValueError("Invalid job envelope")
|
||||
if (
|
||||
type(data["schema_version"]) is not int
|
||||
or data["schema_version"] != 1
|
||||
or data["type"] != "attachment.process.v1"
|
||||
):
|
||||
raise ValueError("Invalid job type")
|
||||
for k in ["job_id", "attachment_id"]:
|
||||
if not isinstance(data[k], str):
|
||||
raise ValueError("Invalid job UUID")
|
||||
UUID(data[k])
|
||||
for k in ["pipeline_version", "attempt"]:
|
||||
if type(data[k]) is not int or data[k] < 1:
|
||||
raise ValueError("Invalid job number")
|
||||
timestamp = data["created_at"]
|
||||
if not isinstance(timestamp, str) or not re.fullmatch(
|
||||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})",
|
||||
timestamp,
|
||||
):
|
||||
raise ValueError("Invalid job timestamp")
|
||||
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
if "traceparent" in data:
|
||||
if not isinstance(data["traceparent"], str) or not re.fullmatch(
|
||||
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"]
|
||||
):
|
||||
raise ValueError("Invalid trace context")
|
||||
if "tracestate" in data and (
|
||||
not isinstance(data["tracestate"], str)
|
||||
or len(data["tracestate"]) > 512
|
||||
):
|
||||
raise ValueError("Invalid trace context")
|
||||
return data
|
||||
|
||||
|
||||
def event(conn, job_id, name, detail=None):
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)",
|
||||
(job_id, name, Jsonb(detail or {})),
|
||||
)
|
||||
|
||||
|
||||
def enqueue(conn, job_id):
|
||||
row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)",
|
||||
(job_id, Jsonb(envelope(row)), row["next_at"]),
|
||||
)
|
||||
|
||||
|
||||
def fault(db, target, phase):
|
||||
if os.getenv("PIPELINE_FAULTS") != "1":
|
||||
return
|
||||
with db.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1",
|
||||
(target, phase),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
conn.execute(
|
||||
"UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s",
|
||||
(row["id"],),
|
||||
)
|
||||
action = row["action"]
|
||||
if action == "crash":
|
||||
os._exit(89)
|
||||
if action == "permanent":
|
||||
raise PermanentFailure("invalid_image")
|
||||
if action == "transient":
|
||||
raise RuntimeError("injected_transient")
|
||||
if action == "hang":
|
||||
time.sleep(120)
|
||||
if action == "wait":
|
||||
end = time.monotonic() + 60
|
||||
while time.monotonic() < end:
|
||||
with db.connection() as conn:
|
||||
released = conn.execute(
|
||||
"SELECT released_at FROM pipeline_faults WHERE id=%s",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if released and released["released_at"]:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
raise RuntimeError("fault_wait_timeout")
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def retry(conn, row, error, *, permanent=False):
|
||||
maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20)
|
||||
failed = permanent or row["attempt"] >= maximum
|
||||
delay = random.uniform(0.25, min(30, 2 ** row["attempt"]))
|
||||
state = "failed" if failed else "queued"
|
||||
code = error if permanent else ("retry_exhausted" if failed else error)
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s",
|
||||
(state, code, delay, row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
|
||||
(
|
||||
state,
|
||||
code if failed else None,
|
||||
row["attachment_id"],
|
||||
row["pipeline_version"],
|
||||
),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
row["id"],
|
||||
"failed" if failed else "retry",
|
||||
{"attempt": row["attempt"], "code": code, "backoff_seconds": delay},
|
||||
)
|
||||
if not failed:
|
||||
enqueue(conn, row["id"])
|
||||
|
||||
|
||||
def watchdog(db):
|
||||
with db.connection() as conn, conn.transaction():
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 16"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
retry(conn, row, "lease_expired")
|
||||
# Recreate a wakeup if its stream entry was lost: never recreate logical jobs.
|
||||
rows = conn.execute(
|
||||
"SELECT j.* FROM jobs j WHERE j.state='queued' AND j.next_at<=clock_timestamp() AND NOT EXISTS(SELECT 1 FROM outbox o WHERE o.job_id=j.id AND (o.published_at IS NULL OR o.published_at>clock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
enqueue(conn, row["id"])
|
||||
|
||||
|
||||
def relay_once(db, queue):
|
||||
watchdog(db)
|
||||
with db.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
validate_job(row["payload"])
|
||||
# WAITAOF must use the SAME connection as XADD (not two pooled calls).
|
||||
with queue.client() as dedicated:
|
||||
entry = dedicated.xadd(
|
||||
STREAM,
|
||||
{"job": json.dumps(row["payload"], separators=(",", ":"))},
|
||||
)
|
||||
confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000)
|
||||
if int(confirmation[0]) < 1:
|
||||
raise RuntimeError("Broker durability confirmation timed out")
|
||||
fault(db, row["job_id"], "relay_after_publish")
|
||||
conn.execute(
|
||||
"UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s",
|
||||
(row["id"],),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
row["job_id"],
|
||||
"published",
|
||||
{"entry": entry, "outbox": row["id"]},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def relay():
|
||||
db, queue = database(), broker()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
if not relay_once(db, queue):
|
||||
time.sleep(0.2)
|
||||
except Exception:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "relay_retry",
|
||||
"code": "dependency_unavailable",
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
queue.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
relay()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import database
|
||||
from src.settings import integer
|
||||
|
||||
|
||||
def reconcile(db, store, *, apply=False, grace=None):
|
||||
grace = (
|
||||
grace
|
||||
if grace is not None
|
||||
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
|
||||
)
|
||||
if grace < 300:
|
||||
raise ValueError("Retention must be at least 300 seconds")
|
||||
actions = []
|
||||
for obj in store.objects():
|
||||
# Recheck each candidate in a bounded transaction, including late PUT age.
|
||||
with db.connection() as conn, conn.transaction():
|
||||
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
|
||||
"now"
|
||||
]
|
||||
if (now - obj.last_modified).total_seconds() < grace:
|
||||
continue
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
|
||||
(obj.object_name,),
|
||||
).fetchone()
|
||||
if reservation and reservation["expires_at"] > now:
|
||||
continue
|
||||
jobs = conn.execute(
|
||||
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
|
||||
(obj.object_name, obj.object_name),
|
||||
).fetchall()
|
||||
if any(j["state"] in {"queued", "processing"} for j in jobs):
|
||||
continue
|
||||
current = conn.execute(
|
||||
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
|
||||
(obj.object_name,),
|
||||
).fetchone()
|
||||
if current:
|
||||
continue
|
||||
if any(
|
||||
j["state"] == "failed"
|
||||
and (now - j["updated_at"]).total_seconds() < grace
|
||||
for j in jobs
|
||||
):
|
||||
continue
|
||||
head = store.head(key=obj.object_name)
|
||||
if (
|
||||
head is None
|
||||
or (now - head.last_modified).total_seconds() < grace
|
||||
):
|
||||
continue
|
||||
action = {
|
||||
"action": "delete_expired_object",
|
||||
"key": obj.object_name,
|
||||
"applied": apply,
|
||||
}
|
||||
if apply:
|
||||
# Taking the reservation lock fences an expired API before deletion.
|
||||
if reservation:
|
||||
conn.execute(
|
||||
"DELETE FROM upload_reservations WHERE id=%s",
|
||||
(reservation["id"],),
|
||||
)
|
||||
store.delete(key=obj.object_name)
|
||||
for job in jobs:
|
||||
if (
|
||||
job["state"] == "failed"
|
||||
and job["source_key"] == obj.object_name
|
||||
):
|
||||
conn.execute(
|
||||
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
|
||||
(job["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
|
||||
(job["attachment_id"], job["pipeline_version"]),
|
||||
)
|
||||
actions.append(action)
|
||||
return {"apply": apply, "retention_seconds": grace, "actions": actions}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p.add_argument("--loop", action="store_true")
|
||||
a = p.parse_args()
|
||||
db, store = database(), ObjectStore()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
print(
|
||||
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
|
||||
)
|
||||
except Exception:
|
||||
if not a.loop:
|
||||
raise
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "cleanup_retry",
|
||||
"code": "dependency_unavailable",
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
if not a.loop:
|
||||
break
|
||||
time.sleep(60)
|
||||
finally:
|
||||
db.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,296 @@
|
||||
import hashlib
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.errors import ForeignKeyViolation, UniqueViolation
|
||||
|
||||
from src.cache import UserCache
|
||||
from src.database import Database
|
||||
from src.domain import Chat, Key, Message, User
|
||||
from src.errors import DomainError, invalid, not_found
|
||||
|
||||
MESSAGE_COLUMNS = (
|
||||
"m.*, ARRAY(SELECT attachment_id FROM message_attachments ma "
|
||||
"WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids"
|
||||
)
|
||||
|
||||
USER_COLUMNS = "id, username, display_name, created_at"
|
||||
|
||||
|
||||
def user_dto(row) -> User:
|
||||
return User(
|
||||
row["id"], row["username"], row["display_name"], row["created_at"]
|
||||
)
|
||||
|
||||
|
||||
def chat_dto(row) -> Chat:
|
||||
return Chat(
|
||||
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
|
||||
)
|
||||
|
||||
|
||||
def message_dto(row) -> Message:
|
||||
return Message(
|
||||
row["id"],
|
||||
row["chat_id"],
|
||||
row["sender_id"],
|
||||
row["text"],
|
||||
row["created_at"],
|
||||
tuple(row.get("attachment_ids", ())),
|
||||
)
|
||||
|
||||
|
||||
class PostgresRepository:
|
||||
def __init__(self, *, database: Database, cache: UserCache):
|
||||
self.database = database
|
||||
self.cache = cache
|
||||
|
||||
def add_user(self, *, user: User, password_hash: str) -> User:
|
||||
try:
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO users(id,username,display_name,"
|
||||
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
|
||||
(
|
||||
user.id,
|
||||
user.username,
|
||||
user.display_name,
|
||||
password_hash,
|
||||
user.created_at,
|
||||
),
|
||||
)
|
||||
except UniqueViolation:
|
||||
raise DomainError(
|
||||
409, "conflict", "Username already exists"
|
||||
) from None
|
||||
return user
|
||||
|
||||
def get_user(self, *, user_id: UUID) -> User | None:
|
||||
cached = self.cache.get(user_id=user_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = user_dto(row)
|
||||
self.cache.put(user=user)
|
||||
return user
|
||||
|
||||
def credentials(self, *, username: str):
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT "
|
||||
+ USER_COLUMNS
|
||||
+ ", password_hash FROM users WHERE username=%s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None, None
|
||||
return user_dto(row), row["password_hash"]
|
||||
|
||||
def username_exists(self, *, username: str) -> bool:
|
||||
with self.database.connection() as conn:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM users WHERE username=%s", (username,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
def add_session(
|
||||
self, *, digest: bytes, user_id: UUID, created_at, expires_at
|
||||
):
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(token_digest,user_id,created_at,"
|
||||
"expires_at) VALUES(%s,%s,%s,%s)",
|
||||
(digest, user_id, created_at, expires_at),
|
||||
)
|
||||
|
||||
def session_user(self, *, digest: bytes) -> UUID | None:
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT user_id FROM sessions WHERE token_digest=%s "
|
||||
"AND expires_at>clock_timestamp()",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
return row["user_id"] if row else None
|
||||
|
||||
def delete_session(self, *, digest: bytes):
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
|
||||
)
|
||||
|
||||
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
|
||||
members = chat.member_ids
|
||||
if actor_id not in members or len(set(members)) != len(members):
|
||||
raise invalid()
|
||||
try:
|
||||
with self.database.connection() as conn, conn.transaction():
|
||||
conn.execute(
|
||||
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
|
||||
(chat.id, chat.title, chat.created_at),
|
||||
)
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO members(chat_id,user_id,position) "
|
||||
"VALUES(%s,%s,%s)",
|
||||
[
|
||||
(chat.id, user_id, position)
|
||||
for position, user_id in enumerate(members)
|
||||
],
|
||||
)
|
||||
except ForeignKeyViolation:
|
||||
raise invalid() from None
|
||||
return chat
|
||||
|
||||
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT c.id,c.title,c.created_at,"
|
||||
"array_agg(m.user_id ORDER BY m.position) member_ids "
|
||||
"FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
|
||||
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
|
||||
(chat_id, actor_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return chat_dto(row)
|
||||
|
||||
def add_message(
|
||||
self, *, message: Message, idempotency_key: str
|
||||
) -> Message:
|
||||
# Область включает пользователя, метод и нормализованный путь чата.
|
||||
scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}"
|
||||
lock = int.from_bytes(
|
||||
hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True
|
||||
)
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"text": message.text,
|
||||
"attachment_ids": [str(x) for x in message.attachment_ids],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).digest()
|
||||
try:
|
||||
with self.database.connection() as conn, conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
prior = conn.execute(
|
||||
"SELECT i.body_hash,i.message_id,"
|
||||
"i.expires_at>clock_timestamp() active "
|
||||
"FROM idempotency i WHERE user_id=%s AND chat_id=%s "
|
||||
"AND key=%s FOR UPDATE",
|
||||
(message.sender_id, message.chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if prior and prior["active"]:
|
||||
if bytes(prior["body_hash"]) != fingerprint:
|
||||
raise DomainError(409, "conflict", "Key already used")
|
||||
row = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.id=%s",
|
||||
(prior["message_id"],),
|
||||
).fetchone()
|
||||
return message_dto(row)
|
||||
if message.attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id,state FROM attachments WHERE chat_id=%s "
|
||||
"AND id=ANY(%s) FOR SHARE",
|
||||
(message.chat_id, list(message.attachment_ids)),
|
||||
).fetchall()
|
||||
if len(rows) != len(message.attachment_ids):
|
||||
raise not_found()
|
||||
if any(row["state"] == "failed" for row in rows):
|
||||
raise DomainError(
|
||||
409, "conflict", "Attachment is not ready"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO messages(id,chat_id,sender_id,"
|
||||
"text,created_at) "
|
||||
"VALUES(%s,%s,%s,%s,%s)",
|
||||
(
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at,
|
||||
),
|
||||
)
|
||||
if message.attachment_ids:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO message_attachments("
|
||||
"message_id,chat_id,"
|
||||
"attachment_id,position) "
|
||||
"VALUES(%s,%s,%s,%s)",
|
||||
[
|
||||
(message.id, message.chat_id, aid, i)
|
||||
for i, aid in enumerate(message.attachment_ids)
|
||||
],
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO idempotency(user_id,chat_id,key,body_hash,"
|
||||
"message_id,expires_at) VALUES(%s,%s,%s,%s,%s,"
|
||||
"clock_timestamp()+interval '24 hours') "
|
||||
"ON CONFLICT(user_id,chat_id,key) DO UPDATE SET "
|
||||
"body_hash=excluded.body_hash,message_id=excluded.message_id,"
|
||||
"expires_at=excluded.expires_at",
|
||||
(
|
||||
message.sender_id,
|
||||
message.chat_id,
|
||||
idempotency_key,
|
||||
fingerprint,
|
||||
message.id,
|
||||
),
|
||||
)
|
||||
except ForeignKeyViolation:
|
||||
raise not_found() from None
|
||||
return message
|
||||
|
||||
def list_chats(
|
||||
self, *, actor_id: UUID, after: Key | None, limit: int
|
||||
) -> list[Chat]:
|
||||
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
|
||||
parameters = (
|
||||
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
|
||||
)
|
||||
with self.database.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
|
||||
"FROM members WHERE chat_id=c.id "
|
||||
"ORDER BY position) member_ids "
|
||||
"FROM chats c JOIN members own ON own.chat_id=c.id "
|
||||
"WHERE own.user_id=%s "
|
||||
+ condition
|
||||
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
parameters,
|
||||
).fetchall()
|
||||
return [chat_dto(row) for row in rows]
|
||||
|
||||
def list_messages(
|
||||
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
|
||||
) -> list[Message]:
|
||||
condition = "AND (m.created_at,m.id)>(%s,%s)" if after else ""
|
||||
parameters = (
|
||||
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
|
||||
)
|
||||
with self.database.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.chat_id=%s "
|
||||
+ condition
|
||||
+ " ORDER BY m.created_at,m.id LIMIT %s",
|
||||
parameters,
|
||||
).fetchall()
|
||||
return [message_dto(row) for row in rows]
|
||||
@@ -0,0 +1,65 @@
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
StrictStr,
|
||||
ValidationInfo,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from src.cursor import parse_uuid
|
||||
|
||||
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
|
||||
|
||||
|
||||
class Input(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator("*", mode="after")
|
||||
@classmethod
|
||||
def nonblank(cls, value: object, info: ValidationInfo) -> object:
|
||||
if isinstance(value, str) and (
|
||||
(not value.strip() and info.field_name != "text")
|
||||
or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
|
||||
):
|
||||
raise ValueError("Invalid text")
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(Input):
|
||||
username: StrictStr
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def username_format(cls, value: str) -> str:
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Invalid username")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(Input):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
|
||||
|
||||
|
||||
class CreateMessage(Input):
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: list[UuidIn] = Field(
|
||||
default_factory=list, max_length=10, strict=True
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content(self):
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("Duplicate attachment")
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Empty message")
|
||||
return self
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.cursor import decode_cursor, encode_cursor
|
||||
from src.domain import Chat, Message, Record, User, order_key, utcnow
|
||||
from src.errors import DomainError, invalid, not_found
|
||||
from src.password import Passwords
|
||||
from src.repository import PostgresRepository
|
||||
|
||||
|
||||
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
|
||||
items = rows[:limit]
|
||||
cursor = (
|
||||
encode_cursor(scope=scope, key=order_key(items[-1]))
|
||||
if len(rows) > limit
|
||||
else None
|
||||
)
|
||||
return {"items": items, "next_cursor": cursor}
|
||||
|
||||
|
||||
def unauthorized(*, basic=False):
|
||||
challenge = (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if basic
|
||||
else 'Bearer realm="messenger"'
|
||||
)
|
||||
return DomainError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def bearer(value):
|
||||
if not value:
|
||||
raise unauthorized()
|
||||
scheme, _, token = value.partition(" ")
|
||||
if scheme.lower() != "bearer" or not re.fullmatch(
|
||||
r"[A-Za-z0-9_-]{32,256}", token
|
||||
):
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
def token_digest(value):
|
||||
return hashlib.sha256(value.encode()).digest()
|
||||
|
||||
|
||||
class MessengerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
repository: PostgresRepository,
|
||||
passwords: Passwords,
|
||||
session_ttl: int,
|
||||
) -> None:
|
||||
self.repository = repository
|
||||
self.passwords = passwords
|
||||
self.session_ttl = session_ttl
|
||||
self.registration_locks = [threading.Lock() for _ in range(64)]
|
||||
|
||||
def identify(self, *, value: str | None) -> User:
|
||||
token = bearer(value)
|
||||
user_id = self.repository.session_user(digest=token_digest(token))
|
||||
user = self.repository.get_user(user_id=user_id) if user_id else None
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
def register(
|
||||
self, *, username: str, display_name: str, password: str
|
||||
) -> User:
|
||||
# Bound lock memory and avoid running Argon2 for every local duplicate.
|
||||
# Different replicas still race safely through PostgreSQL UNIQUE.
|
||||
stripe = hashlib.sha256(username.encode()).digest()[0] % 64
|
||||
lock = self.registration_locks[stripe]
|
||||
if not lock.acquire(timeout=8):
|
||||
raise DomainError(503, "unavailable", "Registration busy")
|
||||
try:
|
||||
if self.repository.username_exists(username=username):
|
||||
raise DomainError(409, "conflict", "Username already exists")
|
||||
encoded = self.passwords.hash(password=password)
|
||||
return self.repository.add_user(
|
||||
user=User(uuid4(), username, display_name, utcnow()),
|
||||
password_hash=encoded,
|
||||
)
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
def login(self, *, authorization: str | None):
|
||||
try:
|
||||
scheme, _, encoded = (authorization or "").partition(" ")
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, separator, password = plain.partition(":")
|
||||
if (
|
||||
not separator
|
||||
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
|
||||
or not 12 <= len(password) <= 128
|
||||
or not password.strip()
|
||||
):
|
||||
raise ValueError
|
||||
except ValueError, UnicodeError:
|
||||
raise unauthorized(basic=True) from None
|
||||
user, password_hash = self.repository.credentials(username=username)
|
||||
if not self.passwords.verify(password=password, encoded=password_hash):
|
||||
raise unauthorized(basic=True)
|
||||
now = utcnow()
|
||||
expires = now + timedelta(seconds=self.session_ttl)
|
||||
token = secrets.token_urlsafe(32)
|
||||
self.repository.add_session(
|
||||
digest=token_digest(token),
|
||||
user_id=user.id,
|
||||
created_at=now,
|
||||
expires_at=expires,
|
||||
)
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires,
|
||||
"user": user,
|
||||
}
|
||||
|
||||
def logout(self, *, authorization: str | None):
|
||||
self.identify(value=authorization)
|
||||
self.repository.delete_session(
|
||||
digest=token_digest(bearer(authorization))
|
||||
)
|
||||
|
||||
def create_chat(
|
||||
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
|
||||
) -> Chat:
|
||||
return self.repository.add_chat(
|
||||
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
|
||||
actor_id=actor_id,
|
||||
)
|
||||
|
||||
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
|
||||
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
|
||||
|
||||
def send(
|
||||
self,
|
||||
*,
|
||||
actor_id: UUID,
|
||||
chat_id: UUID,
|
||||
text: str,
|
||||
idempotency_key: str | None,
|
||||
attachment_ids: list[UUID],
|
||||
) -> Message:
|
||||
if not idempotency_key or not re.fullmatch(
|
||||
r"[A-Za-z0-9._:-]{1,128}", idempotency_key
|
||||
):
|
||||
raise invalid()
|
||||
return self.repository.add_message(
|
||||
message=Message(
|
||||
uuid4(),
|
||||
chat_id,
|
||||
actor_id,
|
||||
text,
|
||||
utcnow(),
|
||||
tuple(attachment_ids),
|
||||
),
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||
scope = f"chats:{actor_id}"
|
||||
after = decode_cursor(value=cursor, scope=scope)
|
||||
rows = self.repository.list_chats(
|
||||
actor_id=actor_id, after=after, limit=limit
|
||||
)
|
||||
return page(rows=rows, limit=limit, scope=scope)
|
||||
|
||||
def history(
|
||||
self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None
|
||||
) -> dict:
|
||||
if actor_id not in chat.member_ids:
|
||||
raise not_found()
|
||||
chat_id = chat.id
|
||||
scope = f"messages:{chat_id}"
|
||||
after = decode_cursor(value=cursor, scope=scope)
|
||||
rows = self.repository.list_messages(
|
||||
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
|
||||
)
|
||||
return page(rows=rows, limit=limit, scope=scope)
|
||||
@@ -0,0 +1,37 @@
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
file = os.environ.get(name + "_FILE")
|
||||
if file:
|
||||
return Path(file).read_text().strip()
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
raise ValueError("Missing required secret setting: " + name)
|
||||
return value
|
||||
|
||||
|
||||
def integer(name: str, default: int, low: int, high: int) -> int:
|
||||
value = int(os.environ.get(name, str(default)))
|
||||
if not low <= value <= high:
|
||||
raise ValueError("Invalid setting: " + name)
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = field(repr=False)
|
||||
cache_url: str = field(repr=False)
|
||||
session_ttl: int = 3600
|
||||
pool_size: int = 10
|
||||
|
||||
@classmethod
|
||||
def load(cls):
|
||||
return cls(
|
||||
secret("DATABASE_URL"),
|
||||
secret("CACHE_URL"),
|
||||
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
|
||||
integer("DB_POOL_SIZE", 10, 1, 32),
|
||||
)
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
"""Two bounded consumers; every decode runs in a killable isolated process."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import multiprocessing
|
||||
import resource
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from src.images import MAX_BYTES, MAX_PIXELS
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import (
|
||||
GROUP,
|
||||
STREAM,
|
||||
PermanentFailure,
|
||||
broker,
|
||||
database,
|
||||
event,
|
||||
fault,
|
||||
retry,
|
||||
validate_job,
|
||||
)
|
||||
from src.settings import integer
|
||||
|
||||
|
||||
def sizes(width, height, bound):
|
||||
longest = max(width, height)
|
||||
return (
|
||||
(width, height)
|
||||
if longest <= bound
|
||||
else (
|
||||
max(1, width * bound // longest),
|
||||
max(1, height * bound // longest),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def make_variants(raw, row):
|
||||
if (
|
||||
len(raw) != row["size_bytes"]
|
||||
or hashlib.sha256(raw).hexdigest() != row["sha256"]
|
||||
):
|
||||
raise PermanentFailure("source_checksum_mismatch")
|
||||
try:
|
||||
with Image.open(io.BytesIO(raw)) as source:
|
||||
if source.width * source.height > MAX_PIXELS:
|
||||
raise PermanentFailure("pixel_limit")
|
||||
if source.format not in {"PNG", "JPEG"}:
|
||||
raise PermanentFailure("invalid_image")
|
||||
source.load()
|
||||
oriented = ImageOps.exif_transpose(source)
|
||||
rgba = oriented.convert("RGBA")
|
||||
rgb = Image.new("RGB", rgba.size, "white")
|
||||
rgb.paste(rgba, mask=rgba.getchannel("A"))
|
||||
rgb.info.clear()
|
||||
yield "original", raw, row["content_type"], source.size
|
||||
thumbnail = rgb.resize(
|
||||
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
|
||||
)
|
||||
crop = ImageOps.fit(
|
||||
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||
)
|
||||
optimized = rgb.resize(
|
||||
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
|
||||
)
|
||||
for name, image, fmt, mime, options in [
|
||||
(
|
||||
"thumbnail",
|
||||
thumbnail,
|
||||
"JPEG",
|
||||
"image/jpeg",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"crop",
|
||||
crop,
|
||||
"JPEG",
|
||||
"image/jpeg",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"optimized",
|
||||
optimized,
|
||||
"WEBP",
|
||||
"image/webp",
|
||||
{
|
||||
"quality": 82 if row["pipeline_version"] == 1 else 76,
|
||||
"method": 4,
|
||||
},
|
||||
),
|
||||
]:
|
||||
output = io.BytesIO()
|
||||
image.info.clear()
|
||||
image.save(output, format=fmt, **options)
|
||||
yield name, output.getvalue(), mime, image.size
|
||||
except (
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
Image.DecompressionBombError,
|
||||
) as exc:
|
||||
raise PermanentFailure("invalid_image") from exc
|
||||
|
||||
|
||||
def process(job_id, fence, sender):
|
||||
db = store = None
|
||||
started = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
||||
db, store = database(), ObjectStore()
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
|
||||
(job_id, fence),
|
||||
).fetchone()
|
||||
if not row:
|
||||
sender.send({"result": "stale"})
|
||||
return
|
||||
# Record limits before fault hooks: a slow external inspector must not
|
||||
# consume the child timeout while attempting to observe the limit.
|
||||
cgroup = Path("/sys/fs/cgroup")
|
||||
limits = {
|
||||
name: (cgroup / name).read_text().strip()
|
||||
if (cgroup / name).exists()
|
||||
else "unavailable"
|
||||
for name in ["memory.max", "cpu.max"]
|
||||
}
|
||||
limits["address_space_bytes"] = list(
|
||||
resource.getrlimit(resource.RLIMIT_AS)
|
||||
)
|
||||
with db.connection() as conn:
|
||||
event(conn, job_id, "worker_resource_limits", limits)
|
||||
fault(db, job_id, "before_effect")
|
||||
upstream = store.get(key=row["source_key"])
|
||||
try:
|
||||
raw = upstream.read(MAX_BYTES + 1)
|
||||
finally:
|
||||
upstream.close()
|
||||
upstream.release_conn()
|
||||
results = []
|
||||
for name, data, mime, (width, height) in make_variants(raw, row):
|
||||
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
store.put(
|
||||
key=key,
|
||||
stream=io.BytesIO(data),
|
||||
size=len(data),
|
||||
content_type=mime,
|
||||
sha256=digest,
|
||||
)
|
||||
results.append((name, key, mime, len(data), digest, width, height))
|
||||
fault(db, job_id, "after_effect")
|
||||
with db.connection() as conn, conn.transaction():
|
||||
current = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
allowed = conn.execute(
|
||||
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
|
||||
(row["attachment_id"],),
|
||||
).fetchone()
|
||||
valid_lease = conn.execute(
|
||||
"SELECT %s::timestamptz>clock_timestamp() live",
|
||||
(current["lease_until"],),
|
||||
).fetchone()["live"]
|
||||
if (
|
||||
current["state"] != "processing"
|
||||
or current["fence"] != fence
|
||||
or not valid_lease
|
||||
or allowed["pipeline_version"] != row["pipeline_version"]
|
||||
):
|
||||
event(
|
||||
conn,
|
||||
job_id,
|
||||
"stale_completion_rejected",
|
||||
{"fence": fence, "version": row["pipeline_version"]},
|
||||
)
|
||||
sender.send({"result": "stale"})
|
||||
return
|
||||
for result in results:
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
|
||||
(row["attachment_id"], row["pipeline_version"], *result),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
|
||||
(row["attachment_id"], row["pipeline_version"], job_id, fence),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(results[0][1], row["attachment_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
job_id,
|
||||
"effect_committed",
|
||||
{
|
||||
"fence": fence,
|
||||
"version": row["pipeline_version"],
|
||||
"variants": 4,
|
||||
},
|
||||
)
|
||||
fault(db, job_id, "after_commit")
|
||||
# Cleanup is independently retryable; it cannot undo the committed effect.
|
||||
if row["source_key"].startswith("objects/staging/"):
|
||||
try:
|
||||
store.delete(key=row["source_key"])
|
||||
except Exception:
|
||||
pass
|
||||
sender.send(
|
||||
{
|
||||
"result": "ready",
|
||||
"seconds": time.monotonic() - started,
|
||||
"cpu_seconds": time.process_time() - cpu,
|
||||
"rss_peak_mib": resource.getrusage(
|
||||
resource.RUSAGE_SELF
|
||||
).ru_maxrss
|
||||
/ 1024,
|
||||
}
|
||||
)
|
||||
except PermanentFailure as exc:
|
||||
sender.send({"result": "permanent", "code": str(exc)})
|
||||
except MemoryError:
|
||||
sender.send({"result": "permanent", "code": "memory_limit"})
|
||||
except Exception:
|
||||
sender.send({"result": "transient", "code": "processing_unavailable"})
|
||||
finally:
|
||||
if store:
|
||||
store.close()
|
||||
if db:
|
||||
db.close()
|
||||
sender.close()
|
||||
|
||||
|
||||
def claim(db, data, consumer):
|
||||
with db.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
|
||||
).fetchone()
|
||||
if (
|
||||
not row
|
||||
or str(row["attachment_id"]) != data["attachment_id"]
|
||||
or row["pipeline_version"] != data["pipeline_version"]
|
||||
):
|
||||
return "obsolete", None
|
||||
if row["state"] in {"ready", "failed", "obsolete"}:
|
||||
return row["state"], row
|
||||
if data["attempt"] < row["attempt"] or (
|
||||
row["state"] == "queued" and data["attempt"] <= row["attempt"]
|
||||
):
|
||||
return "obsolete", row
|
||||
if row["state"] == "processing":
|
||||
return "busy", row
|
||||
if data["attempt"] != row["attempt"] + 1:
|
||||
return "obsolete", row
|
||||
due = conn.execute(
|
||||
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
|
||||
).fetchone()["due"]
|
||||
if not due:
|
||||
return "busy", row
|
||||
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
||||
updated = conn.execute(
|
||||
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
|
||||
(consumer, lease, row["id"]),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
|
||||
(row["attachment_id"], row["pipeline_version"]),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
row["id"],
|
||||
"claimed",
|
||||
{
|
||||
"attempt": updated["attempt"],
|
||||
"fence": updated["fence"],
|
||||
"version": updated["pipeline_version"],
|
||||
},
|
||||
)
|
||||
return "claimed", updated
|
||||
|
||||
|
||||
def ready_effect(db, store, row):
|
||||
with db.connection() as conn:
|
||||
variants = conn.execute(
|
||||
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
|
||||
(row["attachment_id"], row["pipeline_version"]),
|
||||
).fetchall()
|
||||
if len(variants) != 4:
|
||||
return False
|
||||
for variant in variants:
|
||||
head = store.head(key=variant["object_key"])
|
||||
if (
|
||||
head is None
|
||||
or head.size != variant["size_bytes"]
|
||||
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def consume(slot):
|
||||
db, queue, store = database(), broker(), ObjectStore()
|
||||
consumer = str(uuid4()) + f"-{slot}"
|
||||
context = multiprocessing.get_context("spawn")
|
||||
cursor = "0-0"
|
||||
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
|
||||
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
||||
if lease <= timeout:
|
||||
raise ValueError("Lease must exceed child timeout")
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
try:
|
||||
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
reclaimed = queue.xautoclaim(
|
||||
STREAM,
|
||||
GROUP,
|
||||
consumer,
|
||||
# Do not steal an entry from a consumer inside its live budget.
|
||||
min_idle_time=lease * 1000,
|
||||
start_id=cursor,
|
||||
count=1,
|
||||
)
|
||||
cursor, messages = reclaimed[0], reclaimed[1]
|
||||
if not messages:
|
||||
batch = queue.xreadgroup(
|
||||
GROUP, consumer, {STREAM: ">"}, count=1, block=500
|
||||
)
|
||||
messages = batch[0][1] if batch else []
|
||||
for entry, fields in messages:
|
||||
try:
|
||||
data = validate_job(
|
||||
json.loads(fields.get("job", "null"))
|
||||
)
|
||||
except ValueError, TypeError, KeyError:
|
||||
# Malformed notifications cannot become jobs; preserve diagnostic count.
|
||||
with db.connection() as conn:
|
||||
event(conn, None, "invalid_envelope")
|
||||
queue.xack(STREAM, GROUP, entry)
|
||||
queue.xdel(STREAM, entry)
|
||||
continue
|
||||
state, row = claim(db, data, consumer)
|
||||
if state == "busy":
|
||||
continue
|
||||
if state == "ready" and not ready_effect(db, store, row):
|
||||
with db.connection() as conn:
|
||||
event(conn, row["id"], "ready_effect_missing")
|
||||
continue
|
||||
if state in {"ready", "obsolete", "failed"}:
|
||||
with db.connection() as conn:
|
||||
event(
|
||||
conn,
|
||||
data["job_id"],
|
||||
"duplicate_decision",
|
||||
{"state": state, "entry": entry},
|
||||
)
|
||||
if state == "claimed":
|
||||
receiver, sender = context.Pipe(duplex=False)
|
||||
child = context.Process(
|
||||
target=process,
|
||||
args=(str(row["id"]), row["fence"], sender),
|
||||
)
|
||||
child.start()
|
||||
sender.close()
|
||||
child.join(timeout)
|
||||
if child.is_alive():
|
||||
child.kill()
|
||||
child.join(2)
|
||||
result = {
|
||||
"result": "transient",
|
||||
"code": "worker_timeout",
|
||||
}
|
||||
else:
|
||||
try:
|
||||
result = (
|
||||
receiver.recv()
|
||||
if receiver.poll()
|
||||
else {
|
||||
"result": "transient",
|
||||
"code": "worker_crashed",
|
||||
}
|
||||
)
|
||||
except EOFError:
|
||||
result = {
|
||||
"result": "transient",
|
||||
"code": "worker_crashed",
|
||||
}
|
||||
receiver.close()
|
||||
child.close()
|
||||
with db.connection() as conn, conn.transaction():
|
||||
current = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
event(conn, row["id"], "child_finished", result)
|
||||
if (
|
||||
current["state"] == "processing"
|
||||
and current["fence"] == row["fence"]
|
||||
):
|
||||
retry(
|
||||
conn,
|
||||
current,
|
||||
result.get("code", "processing_failed"),
|
||||
permanent=result["result"] == "permanent",
|
||||
)
|
||||
# Commit result or a retry/terminal decision precedes ack.
|
||||
queue.xack(STREAM, GROUP, entry)
|
||||
queue.xdel(STREAM, entry)
|
||||
except Exception:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "worker_retry",
|
||||
"code": "dependency_unavailable",
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
queue.close()
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
|
||||
threads = [
|
||||
threading.Thread(target=consume, args=(i,), daemon=False)
|
||||
for i in range(count)
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
|
||||
CONTRACT = json.loads(
|
||||
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
|
||||
)
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8085")
|
||||
PASSWORD = "Synthetic-Password-🌍-123"
|
||||
TOKENS = {}
|
||||
|
||||
|
||||
def check(result: httpx.Response, status: int, schema: str) -> dict:
|
||||
assert result.status_code == status, result.text
|
||||
assert result.headers["x-request-id"]
|
||||
assert result.headers["x-instance-id"]
|
||||
assert result.headers["content-type"].startswith("application/json")
|
||||
payload = result.json()
|
||||
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
|
||||
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
|
||||
payload
|
||||
)
|
||||
if schema == "Error":
|
||||
assert payload["error"]["request_id"] == result.headers["x-request-id"]
|
||||
assert (
|
||||
payload["error"]["code"]
|
||||
== {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
500: "internal_error",
|
||||
503: "unavailable",
|
||||
}[status]
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def register(client: httpx.Client, **overrides) -> dict:
|
||||
data = {
|
||||
"username": "u_" + uuid4().hex[:24],
|
||||
"display_name": "Алиса 🌍",
|
||||
"password": PASSWORD,
|
||||
}
|
||||
data.update(overrides)
|
||||
user = check(client.post("/api/v1/users", json=data), 201, "User")
|
||||
session = check(
|
||||
client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
auth=httpx.BasicAuth(data["username"], data["password"]),
|
||||
),
|
||||
201,
|
||||
"Session",
|
||||
)
|
||||
TOKENS[user["id"]] = session["token"]
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
with httpx.Client(
|
||||
base_url=BASE_URL, timeout=15, trust_env=False
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def actors(client):
|
||||
return register(client), register(client), register(client)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chat(client, actors):
|
||||
alice, bob, _ = actors
|
||||
return check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(alice),
|
||||
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
|
||||
|
||||
def headers(actor: dict) -> dict:
|
||||
return {
|
||||
"Authorization": "Bearer " + TOKENS[actor["id"]],
|
||||
"Idempotency-Key": uuid4().hex,
|
||||
}
|
||||
|
||||
|
||||
def history(client, actor, chat_id, limit=7):
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
params = {"limit": limit}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
page = check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{chat_id}/messages",
|
||||
headers=headers(actor),
|
||||
params=params,
|
||||
),
|
||||
200,
|
||||
"MessagePage",
|
||||
)
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return found
|
||||
@@ -0,0 +1,412 @@
|
||||
import base64
|
||||
import json
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import BASE_URL, PASSWORD, check, headers, history, register
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
|
||||
def test_health(client, path):
|
||||
assert check(client.get(path), 200, "Health") == {"status": "ok"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
|
||||
)
|
||||
def test_identification(client, value):
|
||||
check(
|
||||
client.get(
|
||||
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
|
||||
),
|
||||
401,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_identity(client, actors):
|
||||
assert (
|
||||
check(
|
||||
client.get("/api/v1/users/me", headers=headers(actors[0])),
|
||||
200,
|
||||
"User",
|
||||
)
|
||||
== actors[0]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data",
|
||||
[
|
||||
{},
|
||||
[],
|
||||
None,
|
||||
{"username": "ABC", "display_name": "A"},
|
||||
{"username": "abc\n", "display_name": "A"},
|
||||
{"username": 123, "display_name": "A"},
|
||||
{"username": "abc", "display_name": " "},
|
||||
{"username": "abc", "display_name": "x" * 101},
|
||||
{"username": "abc", "display_name": "A", "extra": True},
|
||||
{"username": "abc", "display_name": "\ud800"},
|
||||
],
|
||||
)
|
||||
def test_invalid_user(client, data):
|
||||
if isinstance(data, dict):
|
||||
data = {"password": PASSWORD, **data}
|
||||
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
|
||||
)
|
||||
def test_malformed_json(client, raw):
|
||||
check(client.post("/api/v1/users", content=raw), 400, "Error")
|
||||
|
||||
|
||||
def test_auth_before_json(client):
|
||||
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
|
||||
|
||||
|
||||
def test_duplicate_username(client, actors):
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": actors[0]["username"],
|
||||
"display_name": "B",
|
||||
"password": PASSWORD,
|
||||
},
|
||||
),
|
||||
409,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_unicode_codepoints(client, actors, chat):
|
||||
user = register(client, display_name="🌍" * 100)
|
||||
assert user["display_name"] == "🌍" * 100
|
||||
text = "🌍" * 4000
|
||||
result = check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": text},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
assert result["text"] == text
|
||||
assert result["sender_id"] == actors[0]["id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,suffix,body",
|
||||
[
|
||||
("GET", "", None),
|
||||
("GET", "/messages", None),
|
||||
("POST", "/messages", {"text": "hello"}),
|
||||
],
|
||||
)
|
||||
def test_acl(client, actors, chat, method, suffix, body):
|
||||
outsider = headers(actors[2])
|
||||
actual = check(
|
||||
client.request(
|
||||
method,
|
||||
f"/api/v1/chats/{chat['id']}{suffix}",
|
||||
headers=outsider,
|
||||
json=body,
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
missing = check(
|
||||
client.request(
|
||||
method,
|
||||
f"/api/v1/chats/{uuid4()}{suffix}",
|
||||
headers=outsider,
|
||||
json=body,
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
assert actual["error"]["message"] == missing["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
|
||||
def test_bad_message(client, actors, chat, value):
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": value},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_sender_not_client_controlled(client, actors, chat):
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "Hi", "sender_id": actors[1]["id"]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"duplicate",
|
||||
"unknown",
|
||||
"no_caller",
|
||||
"empty",
|
||||
"wrong_type",
|
||||
"bad_uuid",
|
||||
"too_many",
|
||||
"blank_title",
|
||||
],
|
||||
)
|
||||
def test_invalid_chat(client, actors, case):
|
||||
a, b, _ = actors
|
||||
members = {
|
||||
"duplicate": [a["id"], a["id"]],
|
||||
"unknown": [a["id"], str(uuid4())],
|
||||
"no_caller": [b["id"]],
|
||||
"empty": [],
|
||||
"wrong_type": a["id"],
|
||||
"bad_uuid": ["broken"],
|
||||
"too_many": [str(uuid4()) for _ in range(101)],
|
||||
"blank_title": [a["id"]],
|
||||
}[case]
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
json={
|
||||
"title": " " if case == "blank_title" else "Title",
|
||||
"member_ids": members,
|
||||
},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
|
||||
)
|
||||
def test_bad_limit(client, actors, chat, limit):
|
||||
check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
params={"limit": limit},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
|
||||
)
|
||||
def test_bad_cursor(client, actors, chat, cursor):
|
||||
check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
params={"cursor": cursor},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_pagination(client, actors, chat):
|
||||
a, b, outsider = actors
|
||||
sent = []
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
for i in range(23):
|
||||
sent.append(
|
||||
check(
|
||||
client.post(
|
||||
path,
|
||||
headers=headers(a if i % 2 else b),
|
||||
json={"text": f"Msg {i}"},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
)
|
||||
got = history(client, a, chat["id"])
|
||||
assert got == sorted(
|
||||
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
|
||||
)
|
||||
first = check(
|
||||
client.get(path, headers=headers(a), params={"limit": 1}),
|
||||
200,
|
||||
"MessagePage",
|
||||
)
|
||||
second = check(
|
||||
client.get(
|
||||
path,
|
||||
headers=headers(b),
|
||||
params={"limit": 22, "cursor": first["next_cursor"]},
|
||||
),
|
||||
200,
|
||||
"MessagePage",
|
||||
)
|
||||
assert second["items"] == got[1:]
|
||||
assert second["next_cursor"] is None
|
||||
chats = check(
|
||||
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
|
||||
)
|
||||
assert chats["items"] == []
|
||||
other = check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
json={"title": "Other", "member_ids": [a["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
headers=headers(a),
|
||||
params={"cursor": first["next_cursor"]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
|
||||
data = json.loads(
|
||||
base64.urlsafe_b64decode(
|
||||
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
|
||||
)
|
||||
)
|
||||
data["at"] = 17
|
||||
forged = (
|
||||
base64.urlsafe_b64encode(json.dumps(data).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
check(
|
||||
client.get(path, headers=headers(a), params={"cursor": forged}),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_chat_pagination_scope(client, actors):
|
||||
a, b, _ = actors
|
||||
created = []
|
||||
for i in range(4):
|
||||
created.append(
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
)
|
||||
page = check(
|
||||
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
|
||||
200,
|
||||
"ChatPage",
|
||||
)
|
||||
assert page["items"] == created[:2]
|
||||
last = check(
|
||||
client.get(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
params={"limit": 2, "cursor": page["next_cursor"]},
|
||||
),
|
||||
200,
|
||||
"ChatPage",
|
||||
)
|
||||
assert last["items"] == created[2:]
|
||||
assert last["next_cursor"] is None
|
||||
check(
|
||||
client.get(
|
||||
"/api/v1/chats",
|
||||
headers=headers(b),
|
||||
params={"cursor": page["next_cursor"]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_messages(client, actors, chat):
|
||||
barrier = threading.Barrier(10)
|
||||
a = actors[0]
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def worker(i):
|
||||
with httpx.Client(
|
||||
base_url=BASE_URL, timeout=15, trust_env=False
|
||||
) as conn:
|
||||
barrier.wait(timeout=15)
|
||||
return [
|
||||
check(
|
||||
conn.post(
|
||||
path, headers=headers(a), json={"text": f"{i}:{j}"}
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)["id"]
|
||||
for j in range(10)
|
||||
]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
|
||||
rows = history(client, a, chat["id"])
|
||||
assert len(rows) == len(ids) == len(set(ids)) == 100
|
||||
assert set(ids) == {row["id"] for row in rows}
|
||||
assert len({row["text"] for row in rows}) == 100
|
||||
|
||||
|
||||
def test_registration_race(client):
|
||||
username = "race_" + uuid4().hex[:20]
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def worker(_):
|
||||
with httpx.Client(
|
||||
base_url=BASE_URL, timeout=15, trust_env=False
|
||||
) as conn:
|
||||
barrier.wait(timeout=15)
|
||||
result = conn.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": username,
|
||||
"display_name": "Test",
|
||||
"password": PASSWORD,
|
||||
},
|
||||
)
|
||||
check(
|
||||
result,
|
||||
result.status_code,
|
||||
"User" if result.status_code == 201 else "Error",
|
||||
)
|
||||
return result.status_code
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = list(pool.map(worker, range(10)))
|
||||
assert statuses.count(201) == 1
|
||||
assert statuses.count(409) == 9
|
||||
|
||||
|
||||
def test_unknown_path(client):
|
||||
check(client.get("/does-not-exist"), 404, "Error")
|
||||
@@ -0,0 +1,277 @@
|
||||
import hashlib
|
||||
import io
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from conftest import check, headers
|
||||
from PIL import Image
|
||||
|
||||
PNG = (Path(__file__).parents[1] / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def upload(
|
||||
client, actor, chat, *, data=PNG, filename="Фото 🌍.png", mime="image/png"
|
||||
):
|
||||
return client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
headers={**headers(actor), "Content-Type": mime},
|
||||
params={"filename": filename},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
def wait_ready(client, actor, attachment):
|
||||
end = time.monotonic() + 60
|
||||
while time.monotonic() < end:
|
||||
row = check(
|
||||
client.get(
|
||||
f"/api/v1/attachments/{attachment['id']}",
|
||||
headers=headers(actor),
|
||||
),
|
||||
200,
|
||||
"Attachment",
|
||||
)
|
||||
if row["status"] == "ready":
|
||||
return row
|
||||
assert row["status"] != "failed", row
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError("Pipeline readiness")
|
||||
|
||||
|
||||
def test_attachment_roundtrip_acl_and_message(client, actors, chat):
|
||||
attachment = check(upload(client, actors[0], chat), 202, "Attachment")
|
||||
assert attachment["status"] == "queued" and attachment["variants"] == []
|
||||
attachment = wait_ready(client, actors[0], attachment)
|
||||
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
|
||||
assert attachment["size_bytes"] == len(PNG)
|
||||
path = f"/api/v1/attachments/{attachment['id']}"
|
||||
assert (
|
||||
check(client.get(path, headers=headers(actors[1])), 200, "Attachment")
|
||||
== attachment
|
||||
)
|
||||
downloaded = client.get(path + "/content", headers=headers(actors[1]))
|
||||
assert downloaded.status_code == 200 and downloaded.content == PNG
|
||||
assert downloaded.headers["Content-Type"] == "image/png"
|
||||
assert not downloaded.history
|
||||
for suffix in ["", "/content"]:
|
||||
check(
|
||||
client.get(path + suffix, headers=headers(actors[2])), 404, "Error"
|
||||
)
|
||||
check(client.get(path + suffix), 401, "Error")
|
||||
thumb = client.get(
|
||||
path + "/content",
|
||||
headers=headers(actors[1]),
|
||||
params={"variant": "thumbnail"},
|
||||
)
|
||||
assert thumb.status_code == 200
|
||||
with Image.open(io.BytesIO(thumb.content)) as image:
|
||||
assert image.format == "JPEG" and image.size == (256, 160)
|
||||
sent = check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[1]),
|
||||
json={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
assert sent["attachment_ids"] == [attachment["id"]]
|
||||
assert sent["sender_id"] == actors[1]["id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"../x.png",
|
||||
"x/y.png",
|
||||
"x\\y.png",
|
||||
"x\x00.png",
|
||||
"x\x7f.png",
|
||||
"x\n.png",
|
||||
"a" * 256,
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_filename_guard(client, actors, chat, name):
|
||||
check(upload(client, actors[0], chat, filename=name), 400, "Error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,mime,status",
|
||||
[
|
||||
(b"not an image", "image/png", 415),
|
||||
(PNG, "image/jpeg", 415),
|
||||
(PNG, "application/octet-stream", 415),
|
||||
(PNG[:25], "image/png", 415),
|
||||
(b"", "image/png", 415),
|
||||
(b"x" * (10 * 1024 * 1024 + 1), "image/png", 413),
|
||||
],
|
||||
)
|
||||
def test_image_guards(client, actors, chat, data, mime, status):
|
||||
r = upload(client, actors[0], chat, data=data, mime=mime)
|
||||
check(r, status, "Error")
|
||||
|
||||
|
||||
def test_jpeg_original_unchanged(client, actors, chat):
|
||||
stream = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (30, 60, 90)).save(stream, format="JPEG")
|
||||
jpeg = stream.getvalue()
|
||||
attachment = check(
|
||||
upload(
|
||||
client,
|
||||
actors[0],
|
||||
chat,
|
||||
data=jpeg,
|
||||
filename="camera.jpg",
|
||||
mime="image/jpeg",
|
||||
),
|
||||
202,
|
||||
"Attachment",
|
||||
)
|
||||
attachment = wait_ready(client, actors[0], attachment)
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
headers=headers(actors[1]),
|
||||
)
|
||||
assert r.content == jpeg
|
||||
|
||||
|
||||
def test_cross_chat_link_and_unknown(client, actors, chat):
|
||||
a = check(upload(client, actors[0], chat), 202, "Attachment")
|
||||
other = check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "Other", "member_ids": [actors[0]["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
for identity in [a["id"], str(uuid4())]:
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "", "attachment_ids": [identity]},
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "", "attachment_ids": [a["id"], a["id"]]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_default_attachments_are_semantically_equal(client, actors, chat):
|
||||
h = headers(actors[0])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
first = check(
|
||||
client.post(path, headers=h, json={"text": "Semantic default"}),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
second = check(
|
||||
client.post(
|
||||
path,
|
||||
headers=h,
|
||||
json={"attachment_ids": [], "text": "Semantic default"},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
assert first == second and first["attachment_ids"] == []
|
||||
|
||||
|
||||
def test_pixel_limit_before_decode(client, actors, chat):
|
||||
# Compressible synthetic image stays below10MiB but exceeds40M pixels.
|
||||
stream = io.BytesIO()
|
||||
Image.new("1", (8000, 5001)).save(stream, format="PNG")
|
||||
assert len(stream.getvalue()) < 10 * 1024 * 1024
|
||||
r = upload(client, actors[0], chat, data=stream.getvalue())
|
||||
assert r.status_code == 413
|
||||
|
||||
|
||||
def test_chunked_body_limit_without_content_length(client, actors, chat):
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
headers={**headers(actors[0]), "Content-Type": "image/png"},
|
||||
params={"filename": "chunked.png"},
|
||||
content=iter([b"x" * (10 * 1024 * 1024 + 1)]),
|
||||
)
|
||||
assert r.request.headers.get("content-length") is None
|
||||
assert r.status_code == 413
|
||||
assert r.json()["error"]["code"] == "payload_too_large"
|
||||
assert r.json()["error"]["request_id"] == r.headers["x-request-id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [(1, 17), (320, 200), (1600, 900)])
|
||||
def test_variant_geometry_and_alpha(client, actors, chat, size):
|
||||
stream = io.BytesIO()
|
||||
Image.new("RGBA", size, (0, 0, 0, 0)).save(stream, format="PNG")
|
||||
a = check(
|
||||
upload(client, actors[0], chat, data=stream.getvalue()),
|
||||
202,
|
||||
"Attachment",
|
||||
)
|
||||
a = wait_ready(client, actors[0], a)
|
||||
for v in a["variants"]:
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=headers(actors[1]),
|
||||
params={"variant": v["name"]},
|
||||
)
|
||||
assert len(r.content) == v["size_bytes"]
|
||||
assert hashlib.sha256(r.content).hexdigest() == v["sha256"]
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert image.size == (v["width"], v["height"])
|
||||
assert not image.getexif()
|
||||
assert min(image.convert("RGB").getpixel((0, 0))) >= 250
|
||||
if v["name"] == "crop":
|
||||
assert image.size == (128, 128)
|
||||
else:
|
||||
bound = 256 if v["name"] == "thumbnail" else 1280
|
||||
assert max(image.size) <= bound
|
||||
assert image.width <= size[0] and image.height <= size[1]
|
||||
|
||||
|
||||
def test_exif_orientation_applied_but_original_preserved(client, actors, chat):
|
||||
stream = io.BytesIO()
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
exif[270] = "Synthetic metadata must be removed"
|
||||
Image.new("RGB", (320, 200), "red").save(stream, format="JPEG", exif=exif)
|
||||
raw = stream.getvalue()
|
||||
a = check(
|
||||
upload(
|
||||
client,
|
||||
actors[0],
|
||||
chat,
|
||||
data=raw,
|
||||
mime="image/jpeg",
|
||||
filename="exif.jpg",
|
||||
),
|
||||
202,
|
||||
"Attachment",
|
||||
)
|
||||
a = wait_ready(client, actors[0], a)
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=headers(actors[1])
|
||||
)
|
||||
assert r.content == raw
|
||||
v = next(v for v in a["variants"] if v["name"] == "thumbnail")
|
||||
assert (v["width"], v["height"]) == (160, 256)
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=headers(actors[1]),
|
||||
params={"variant": "thumbnail"},
|
||||
)
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert image.size == (160, 256) and not image.getexif()
|
||||
@@ -0,0 +1,26 @@
|
||||
import json
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from conftest import check, headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
|
||||
def test_only_utf8_json(client, encoding):
|
||||
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
|
||||
payload["password"] = "Synthetic-encoding-test-123"
|
||||
result = client.post(
|
||||
"/api/v1/users",
|
||||
content=json.dumps(payload).encode(encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
check(result, 400, "Error")
|
||||
|
||||
|
||||
def test_leading_zero_limit(client, actors, chat):
|
||||
result = client.get(
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
headers=headers(actors[0]),
|
||||
params={"limit": "0001"},
|
||||
)
|
||||
check(result, 200, "MessagePage")
|
||||
@@ -0,0 +1,23 @@
|
||||
from conftest import check, headers
|
||||
|
||||
|
||||
def test_history_acl_precedes_query_validation(client, actors, chat):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
check(
|
||||
client.get(
|
||||
path,
|
||||
headers=headers(actors[2]),
|
||||
params={"limit": 0, "cursor": "broken"},
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
check(
|
||||
client.get(
|
||||
path,
|
||||
headers=headers(actors[0]),
|
||||
params={"limit": 0, "cursor": "broken"},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import BASE_URL, check, headers, history
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "a b", "x" * 129, "a/b"])
|
||||
def test_key_required_and_valid(client, actors, chat, key):
|
||||
h = headers(actors[0])
|
||||
h.pop("Idempotency-Key")
|
||||
if key is not None:
|
||||
h["Idempotency-Key"] = key
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=h,
|
||||
json={"text": "Key"},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_replay_across_instances(client, actors, chat):
|
||||
h = headers(actors[0])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def send(_):
|
||||
with httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15) as c:
|
||||
r = c.post(path, headers=h, json={"text": "Один результат 🌍"})
|
||||
return check(r, 201, "Message"), r.headers["x-instance-id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(send, range(30)))
|
||||
assert len({r[0]["id"] for r in results}) == 1
|
||||
assert len({r[1] for r in results}) >= 2
|
||||
assert history(client, actors[0], chat["id"]) == [results[0][0]]
|
||||
check(
|
||||
client.post(path, headers=h, json={"text": "Другой текст"}),
|
||||
409,
|
||||
"Error",
|
||||
)
|
||||
# Другой пользователь может независимо использовать тот же ключ.
|
||||
other = headers(actors[1])
|
||||
other["Idempotency-Key"] = h["Idempotency-Key"]
|
||||
second = check(
|
||||
client.post(path, headers=other, json={"text": "Боб"}), 201, "Message"
|
||||
)
|
||||
assert second["id"] != results[0][0]["id"]
|
||||
|
||||
|
||||
def test_replay_rechecks_acl(client, actors, chat):
|
||||
h = headers(actors[0])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
check(
|
||||
client.post(path, headers=h, json={"text": "Existing"}), 201, "Message"
|
||||
)
|
||||
outsider = headers(actors[2])
|
||||
outsider["Idempotency-Key"] = h["Idempotency-Key"]
|
||||
check(
|
||||
client.post(path, headers=outsider, json={"text": "Existing"}),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
check(
|
||||
client.post(
|
||||
path,
|
||||
headers={"Idempotency-Key": uuid4().hex},
|
||||
json={"text": "Existing"},
|
||||
),
|
||||
401,
|
||||
"Error",
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import PASSWORD, TOKENS, check, headers, register
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
|
||||
)
|
||||
def test_password_validation(client, password):
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": "pwd_" + uuid4().hex[:20],
|
||||
"display_name": "Test",
|
||||
"password": password,
|
||||
},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_full_utf8_password(client):
|
||||
password = "🌍" * 127 + ":"
|
||||
user = register(client, password=password)
|
||||
assert (
|
||||
check(
|
||||
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
|
||||
)
|
||||
== user
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
|
||||
)
|
||||
def test_bad_basic(client, authorization):
|
||||
result = client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": authorization} if authorization else {},
|
||||
)
|
||||
check(result, 401, "Error")
|
||||
assert (
|
||||
result.headers["www-authenticate"]
|
||||
== 'Basic realm="messenger", charset="UTF-8"'
|
||||
)
|
||||
|
||||
|
||||
def test_wrong_password_and_unknown_user(client):
|
||||
user = register(client)
|
||||
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
|
||||
result = client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
|
||||
)
|
||||
data = check(result, 401, "Error")
|
||||
assert data["error"]["message"] == "Authentication required"
|
||||
|
||||
|
||||
def test_logout_is_not_revived_by_cache(client):
|
||||
user = register(client)
|
||||
auth = headers(user)
|
||||
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
|
||||
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
|
||||
assert result.status_code == 204 and result.content == b""
|
||||
assert result.headers["x-request-id"]
|
||||
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
|
||||
check(
|
||||
client.delete("/api/v1/auth/sessions/current", headers=auth),
|
||||
401,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_only_current_session_revoked(client):
|
||||
user = register(client)
|
||||
another = check(
|
||||
client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
auth=httpx.BasicAuth(user["username"], PASSWORD),
|
||||
),
|
||||
201,
|
||||
"Session",
|
||||
)
|
||||
assert another["token"] != TOKENS[user["id"]]
|
||||
assert len(another["token"]) >= 32
|
||||
assert (
|
||||
client.delete(
|
||||
"/api/v1/auth/sessions/current", headers=headers(user)
|
||||
).status_code
|
||||
== 204
|
||||
)
|
||||
result = client.get(
|
||||
"/api/v1/users/me",
|
||||
headers={"Authorization": "Bearer " + another["token"]},
|
||||
)
|
||||
assert check(result, 200, "User") == user
|
||||
|
||||
|
||||
def test_bearer_source_and_ignored_user_header(client, actors):
|
||||
alice, bob, _ = actors
|
||||
auth = {**headers(alice), "X-User-Id": bob["id"]}
|
||||
assert (
|
||||
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
|
||||
== alice["id"]
|
||||
)
|
||||
result = client.get(
|
||||
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
|
||||
)
|
||||
check(result, 401, "Error")
|
||||
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'
|
||||
@@ -0,0 +1,119 @@
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
|
||||
from src.pipeline import validate_job
|
||||
|
||||
|
||||
def example():
|
||||
return dict(
|
||||
schema_version=1,
|
||||
job_id=str(uuid4()),
|
||||
type="attachment.process.v1",
|
||||
attachment_id=str(uuid4()),
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
created_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def test_schema_and_runtime_agree_on_valid_job():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
schema = json.loads(Path("contracts/jobs.schema.json").read_text())
|
||||
for version in [1, 2]:
|
||||
data = example()
|
||||
data["pipeline_version"] = version
|
||||
data.update(
|
||||
traceparent="00-" + "a" * 32 + "-" + "b" * 16 + "-01",
|
||||
tracestate="vendor=opaque",
|
||||
)
|
||||
Draft202012Validator(schema, format_checker=FormatChecker()).validate(
|
||||
data
|
||||
)
|
||||
assert validate_job(data) == data
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("schema_version", True),
|
||||
("schema_version", 2),
|
||||
("type", "other"),
|
||||
("job_id", 123),
|
||||
("job_id", "broken"),
|
||||
("attachment_id", None),
|
||||
("pipeline_version", 0),
|
||||
("pipeline_version", True),
|
||||
("attempt", 0),
|
||||
("attempt", 1.0),
|
||||
("created_at", "2026-01-01"),
|
||||
("created_at", "2026-01-01T12:00:00"),
|
||||
("created_at", None),
|
||||
("traceparent", "not-w3c"),
|
||||
("tracestate", "x" * 513),
|
||||
("url", "http://invalid.example"),
|
||||
("bytes", "secret-contents"),
|
||||
("filename", "../../bad"),
|
||||
],
|
||||
)
|
||||
def test_invalid_notification_is_rejected(field, value):
|
||||
data = copy.deepcopy(example())
|
||||
data[field] = value
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
validate_job(data)
|
||||
|
||||
|
||||
def test_worker_rejects_oversized_pixels_before_decode():
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from src.pipeline import PermanentFailure
|
||||
from src.worker import make_variants
|
||||
|
||||
stream = io.BytesIO()
|
||||
Image.new("1", (8000, 5001)).save(stream, format="PNG")
|
||||
raw = stream.getvalue()
|
||||
row = dict(
|
||||
size_bytes=len(raw),
|
||||
sha256=hashlib.sha256(raw).hexdigest(),
|
||||
content_type="image/png",
|
||||
pipeline_version=1,
|
||||
)
|
||||
with (
|
||||
pytest.warns(Image.DecompressionBombWarning),
|
||||
pytest.raises(PermanentFailure, match="pixel_limit"),
|
||||
):
|
||||
list(make_variants(raw, row))
|
||||
|
||||
|
||||
def test_worker_rejects_changed_source_checksum():
|
||||
from src.pipeline import PermanentFailure
|
||||
from src.worker import make_variants
|
||||
|
||||
with pytest.raises(PermanentFailure, match="source_checksum_mismatch"):
|
||||
list(make_variants(b"changed", dict(size_bytes=7, sha256="0" * 64)))
|
||||
|
||||
|
||||
def test_worker_rejects_invalid_image_as_permanent():
|
||||
import hashlib
|
||||
|
||||
from src.pipeline import PermanentFailure
|
||||
from src.worker import make_variants
|
||||
|
||||
raw = b"not an image"
|
||||
with pytest.raises(PermanentFailure, match="invalid_image"):
|
||||
list(
|
||||
make_variants(
|
||||
raw,
|
||||
dict(
|
||||
size_bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()
|
||||
),
|
||||
)
|
||||
)
|
||||
Reference in New Issue
Block a user