ЛР 5 — БВТ2301 — Маслов — на 5 #70

Closed
wh1tet3a wants to merge 11 commits from wh1tet3a/mtusi2026lab5:solution/bvt2301-maslov into main
85 changed files with 41076 additions and 45 deletions
+17
View File
@@ -18,3 +18,20 @@ __pycache__
node_modules
evidence
.DS_Store
**/__pycache__
**/*.py[cod]
.work
.pytest_cache
.ruff_cache
tests
student-tests
research
scripts
contracts
*.md
Makefile
compose.yaml
.work*
student-unit
+11 -4
View File
@@ -13,13 +13,20 @@ API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_BUCKET=messenger
S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key
S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
PUBLIC_PORT=8085
API_CPUS=0.5
API_MEMORY=768m
DB_POOL_SIZE=10
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=23
STAGING_RETENTION_SECONDS=86400
+17 -15
View File
@@ -1,18 +1,20 @@
## Лабораторная 5
## Лабораторная5
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Маслов, БВТ2301. Цель5, трек2: идемпотентность versioned image pipeline.
Собственная ЛР4 перенесена с объявленным переходом upload201→202queued.
Реализованы durable private S3 staging/reservation, atomic PG job+outbox,
persistent Valkey Streams/AOF/WAITAOF, relay/watchdog, отдельный bounded worker,
4variants, version/fence/live-lease CAS, finite retry/jitter/failed registry,
audited replay/newversion и safe retention cleanup.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Контракт и публичные tests сохранены. Запуск, схема, fault methodology,
результаты и ответы защиты — REPORT.md/SCHEMA.md.112HTTP и22unit прошли;
полный clean-clone-final PASS:19fault cases, cluster/rolling/persistence,
real-age cleanup,6runtime source/OCI checks и fsck. Строгое pending-before-SIGKILL
окно COMMIT→ACK подтверждено дважды. Промежуточные failed runs сохранены.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Ограничения: одинhost/SPOFs, нет production TLS/backup/global admission quota;
HEAD не periodic byte audit; at-least-once с идемпотентным эффектом,
не exactly-once delivery; удалённый CI не подтверждён.
При разработке кода, проверок и отчёта использована помощь ИИ. Источники —
teacher contracts, собственная ЛР4, pinned libraries и официальные Valkey/Pillow docs.
+27
View File
@@ -0,0 +1,27 @@
name: Application integration
on: [push, pull_request, workflow_dispatch]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 90
env:
COMPOSE_FILE: compose.yaml:compose.test.yaml
NO_PROXY: localhost,127.0.0.1
no_proxy: localhost,127.0.0.1
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v6
with:
version: 0.13.7
- run: uv sync --locked
- run: make bootstrap check validate lint unit PYTHON=.venv/bin/python
- run: VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
- run: make test test-student PYTHON=.venv/bin/python BASE_URL=http://127.0.0.1:8085
- run: uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-ci-seed.json
- run: uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-ci.json
- run: uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-ci-seed.json --output evidence/gc-ci.json
- run: uv run --locked python -m scripts.verify_cluster --output evidence/cluster-ci.json
- if: always()
run: docker compose down --volumes --remove-orphans
+5
View File
@@ -30,3 +30,8 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.work/
.work-*.txt
.pytest_cache/
.ruff_cache/
+22
View File
@@ -0,0 +1,22 @@
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=uv /uv /usr/local/bin/uv
WORKDIR /app
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ARG VCS_REF=unknown
LABEL org.opencontainers.image.revision=$VCS_REF
WORKDIR /app
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY --from=builder /app/.venv /app/.venv
COPY src /app/src
COPY migrations /app/migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+23
View File
@@ -28,3 +28,26 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
UV ?= uv
BASE_URL = http://127.0.0.1:8085
.PHONY: bootstrap up down lint test-student verify
bootstrap:
$(PYTHON) scripts/bootstrap.py
up: bootstrap
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
down:
docker compose down
lint:
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
test-student:
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
verify:
$(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json
.PHONY: unit reconcile
unit:
PYTHONPATH=. $(UV) run --locked pytest student-unit -q
reconcile:
docker compose run --rm --no-deps reconciler python -m src.reconcile
+397 -22
View File
@@ -1,37 +1,412 @@
# Отчёт — лабораторная 5
# Лабораторная 5 — Маслов, БВТ2301
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Цель: **5, трек 2 — идемпотентность versioned pipeline**. Собственная ЛР4
перенесена в независимую историю текущего teacher skeleton; обязательные
contracts/tests/check/validate сохранены. API принимает исходник, а отдельный
worker публикует original/thumbnail/crop/optimized. Исследование проверяет
дубли, reorder и реальные SIGKILL, а не моделирует их ответами mock.
## Запуск из чистого clone
Полный повтор из чистого clone — **PASS**, проверенный runtime source
`442163b0f0a549ef065cb13ee39b5ca2f8580c18`. Прошли112HTTP,22unit, public smoke,
19fault cases, cluster/rolling/persistence и cleanup с настоящим возрастом S3.
Raw receipt: [clean-clone-final.json](evidence/clean-clone-final.json).
Предыдущие неполные запуски сохранены и разобраны ниже; ошибочный ранний
HTTP-порт отмечен INVALID_ENVIRONMENT и не считается проверкой API.
Удалённый CI отдельно не подтверждён.
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
## Воспроизведение
## Архитектура и решения
Нужны Git, Docker Desktop с Compose v2, uv0.13.7. Версии Python3.14.7,
PostgreSQL18.3, Valkey8.1.3, HAProxy3.2.1, Pillow13.0.0 закреплены;
MinIO/mc собираются из точных upstream commits в infra/s3.Dockerfile,
Go module checksum проверяется GoSumDB. Сборка требует доступа к registries,
Go module proxy и PyPI; наличие сети нельзя заменить уже установленной БД.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```sh
uv sync --locked
make bootstrap
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
make check validate PYTHON=.venv/bin/python
make lint unit
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 make test PYTHON=.venv/bin/python
BASE_URL=http://127.0.0.1:8085 make test-student
```
## Доказательства критериев
URL: http://127.0.0.1:8085. API не имеет persistent volume/host port.
Bootstrap создаёт только отсутствующие secrets и .env; не печатает значения,
не перезаписывает существующие. `secrets/`0700, файлы0644 нужны UID10001
через readonly Docker secrets mounts; сам каталог недоступен другим users.
`docker compose down` сохраняет volumes. Удаление volumes — отдельное
осознанное действие над своим синтетическим стендом.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
Для автоматических отказов включается только явный test override:
```sh
export COMPOSE_FILE=compose.yaml:compose.test.yaml
export NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-seed-repeat.json
uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json --repeats 2
uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-seed-repeat.json --output evidence/gc-repeat.json
uv run --locked python -m scripts.verify_cluster --output evidence/cluster-repeat.json
```
Проверки, меняющие состояние контейнеров, выполняются последовательно.
Seed и результат — разные файлы; нельзя использовать seed с уже удалённым
orphan. Verify ожидает настоящий возраст S3 не менее302s; часы/LastModified
не подменяются. Для нового полного чистого повторения:
```sh
# Перед стартом весь проверяемый source должен быть закоммичен и git clean.
uv run --locked python scripts/verify_clean.py --output evidence/clean-clone-repeat.json
```
Этот verifier создаёт отдельные secrets/env/volumes/image/project, собирает,
запускает static/unit/public smoke/HTTP, cluster, всю fault matrix и cleanup,
сверяет source hashes и OCI revision обоих API, worker, relay, reconciler и
тестового controller; после удаляет только временный проект и его volumes.
## Границы устойчивости
1. API проверяет фактические bytes≤10MiB, MIME PNG/JPEG, декодирование,
filename и ≤40M pixels до полной распаковки. Временный spool≤1MiB в RAM,
затем bounded tmpfs; upload slots2/API, decode slot1/API, срок запроса20s.
2. Устойчивый upload reservation120s COMMIT предшествует PUT в приватный
`objects/staging/<UUID>/original`. После PUT одна PG transaction проверяет
reservation с FOR UPDATE, создаёт Attachment queued + logical Job + Outbox,
удаляет reservation и COMMIT. Только затем202 с захваченным queued-снимком.
Быстрый worker не превращает upload-ответ в ready. Сам upload не имеет
Idempotency-Key: потерянный HTTP-ответ нельзя безопасно повторять вслепую.
3. API зависит от PG и S3 для ready, cache используется как необязательный
read-through user cache. Broker не критичен для синхронного приёма:
работа остаётся в outbox. Relay имеет PG+broker secrets, не S3/root keys.
4. Relay читает due outbox FOR UPDATE SKIP LOCKED, XADD и WAITAOF1/0/2000
**на одной выделенной connection**. Подтверждён fsync локального AOF;
только затем published_at/COMMIT в PG. Crash в этом окне даёт дубль.
5. Отдельный Valkey Streams broker с persistent volume, AOF appendfsync always,
noeviction. Disposable cache — другой сервис/volume/policy. Нет Pub/Sub.
6. Worker consumer group, COUNT=1 на каждый из2 слотов; XAUTOCLAIM возвращает
зависшие PEL entries. PG атомарно переводит queued→processing, увеличивает
attempt/fence, сохраняет owner/lease. Duplicate active оставляется pending;
ready duplicate делает HEAD всех4вариантов/size/SHA metadata и ACK без resize.
7. Child получает ID/fence, trusted source берётся из PG, не из произвольного
URL или filename. До обработки проверяется исходный size/SHA. Все4 PUT идут
в `objects/<attachment>/v<version>/<variant>`. Original побайтно неизменён.
8. В одной PG transaction worker проверяет state/version/fence/live lease,
публикует metadata всех вариантов, unique effect_publication и ready.
Старый child может дописать свой version-prefix, но не подтвердить новый
результат. Стабильный алгоритм нельзя менять под тем же pipeline_version.
9. ACK только после устойчивой ready, failed/obsolete или решения queued retry.
Watchdog relay восстанавливает expired leases; pending outbox сохраняет retry.
Потерянные wakeups queued jobs переиздаются через15s без создания logical job.
## Состояния и варианты
GET показывает queued/processing/ready/failed. Queued/processing можно связать
с Message; failed —409. Все metadata/content проходят Bearer+membership,
чужой/неизвестный UUID —404, content до ready —409. У ready все3 производных
варианта и error_code=null; size/content_type/SHA описывают original.
Прямого redirect/public S3 URL нет. Atomic Message+Idempotency-Key сохранены.
EXIF orientation применяется; RGBA компонуется на белом. Thumbnail JPEG
вписан≤256×256 без увеличения, floor/min1;320×200→256×160. Crop — центральный
JPEG128×128 с допустимым увеличением маленького исходника. JPEGquality85,
subsampling0,LANCZOS. Optimized — WebP≤1280×1280 без увеличения, quality82(v1)
или76(v2), method4. EXIF/info удаляются из производных; metadata original
сохраняются в исходных bytes. Маленький WebP не обязательно меньше original.
Job соответствует contracts/jobs.schema.json: schema_version1,
type attachment.process.v1, UUID job/attachment, version, attempt>=1, RFC3339
created_at; возможен W3C carrier. Bytes/credentials/paths/URL запрещены.
Broker redelivery не меняет job_id; finite replay сохраняет job_id и увеличивает
cycle; новая pipeline version получает новый job_id. Idempotency effect key —
(attachment_id,pipeline_version,variant), PK в PG и детерминированный S3 key.
## Ограничение ресурсов и повторы
| Роль | CPU | RAM | Ограничение работы |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| API каждая из2 |0.5|768MiB|2uploads,1decode,128MiB tmpfs|
| Worker |2|1536MiB|2child,COUNT1/slot,child RLIMIT_AS768MiB|
| Relay |0.25|192MiB|один bounded publish/transaction|
| Broker |0.5|256MiB|192MiB maxmemory,noeviction,AOF volume|
| PostgreSQL |1|512MiB|pool10/API,4/worker-slot|
| S3 |1|1024MiB|persistent volume,private bucket|
| Reconciler |0.25|256MiB|inventory проход/60s|
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Normal timeout20s / lease23s; test override12s/15s даёт время контроллеру
завершить v2 в другой slot пока v1 удерживается после PUT. Child действительно
killable: parent убивает процесс по timeout, а не лишь отменяет future/thread.
Весь container ограничен cgroup; лимит40M pixels не обещает, что всякий такой
файл уложится в память и срок — memory_limit/timeout приводят к safe failed/retry.
## Эксперименты и отказы
Максимум5attempts; временные ошибки/смерть worker/timeout → persisted next_at
с jitter uniform0.25…min(30,2^attempt)s. Постоянные decode/checksum/memory ошибки
сразу failed; временные после5→retry_exhausted. Публичные error_code безопасны,
без stack/SQL/key. Реестр failed и audited replay доступны только оператору:
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
```sh
docker compose exec worker python -m src.operator failed
docker compose exec worker python -m src.operator replay --job-id UUID --operator maslov --reason 'Synthetic recovery demonstration'
docker compose exec worker python -m src.operator version --job-id UUID --version 2 --operator maslov --reason 'Upgrade pipeline'
docker compose run --rm --no-deps reconciler python -m src.reconcile
docker compose run --rm --no-deps reconciler python -m src.reconcile --apply
```
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
Ready replay отвергается. Failed replay проверяет наличие source, оставляет
audit и новый finite cycle. После source_expired нельзя обещать обработку
отсутствующих bytes. New-version поддерживает реализованный переход1→2.
## Retention
Staging удаляется после безопасного COMMIT ready; сбой удаления не отменяет
результат, последующий GC уберёт остаток. Failed/unconfirmed source и orphan
живут минимум retention86400s, конфигурационный hard minimum300s. Cleanup
по умолчанию dry-run; --apply явный. Active queued/processing source и prefix,
действующий reservation и current published variants защищены. Перед delete
повторяются PG lock/recheck и HEAD реального возраста. Expired reservation
удаляется под lock; API не сможет поздно подтвердить удалённый source.
Старые версии после успешной новой могут удаляться; metadata/audit сохраняются.
Failed без source остаётся failed/source_expired и replay отвергается.
Текущий original не удаляется лишь из-за отсутствия Message links.
## Исследование: гипотеза и инварианты
Гипотеза: durable intent/outbox + PG version/fence/lease CAS + immutable
version keys сохраняют подтверждённую работу и не допускают publication старой
версии поверх новой при at-least-once delivery.
Проверяемые инварианты: каждый202 имеет durable Job; все принятые matrix jobs
доходят до ready или наблюдаемого failed; финальный ready имеет ровно4 variant
rows/keys и1effect_publication на версию; original SHA равен загрузке;
old version не меняет hashes текущей; конечный retry/replay наблюдаемы;
cleanup не удаляет current/active/fresh. До handoff crash не получает202.
`scripts/verify_pipeline.py --repeats2` использует отдельный test controller
с PG/S3/broker secrets (API не получает broker secrets), single-use hooks в PG
с durable hit_at, реальные container SIGKILL и повторный запуск. Hooks по
умолчанию выключены, публичного endpoint для них нет. Проверяются:
source-before-handoff, COMMIT-before-publish, confirmed broker SIGKILL/volume,
relay XADD-before-PG-marker, worker before_effect/after_effect/after_commit
каждый дважды; transient, timeout, permanent/replay,5attempt exhaustion/replay,
duplicates/reorder, held stalev1 послеv2 и старые notifications; malformed
notification, private IAM/anonymous denial и другая API после uploader stop.
Raw evidence сохраняет IDs, attempts, fences, events,4keys/checksums и
publications. Worker child_finished даёт elapsed/CPU/peakRSS; это длительность
child, не полный queue lag. Исследование про корректность, не максимальный RPS.
## Вопросы защиты
**Publisher confirm и consumer ACK?** Первый подтверждает durable приём брокером
(здесь локальный AOF fsync через WAITAOF), второй — устойчивое завершение или
сохранённое решение о повторе у consumer. Между PG/Valkey/S3 нет общей transaction.
**Что уже сохранено к202?** Полный проверенный original в private persistent
S3 staging плюс attachment/job/outbox в одной подтверждённой PG transaction.
Смерть API/worker не удаляет намерение. Перед202 тяжёлого resize нет.
**Почему дубли нормальны?** Producer/consumer могут потерять ответ после
успешного действия. Отправитель повторяет, поскольку не знает исход. Logical
job_id и effect tuple/version/fence позволяют повторить доставку безопасно.
Ровно одна publication в PG не означает exactly-once delivery в broker.
**Как отличить зависший worker от долгого?** Совершенно точно нельзя: lease —
выбранный бюджет. Parent timeout убивает child; watchdog восстанавливает работу
после lease, новый fence делает старое завершение недействительным. Ограничение
slots/COUNT сдерживает конкуренцию. Возможен лишний PUT, но не старый COMMIT.
## Источники, помощь, ограничения
Использованы текущие teacher contracts, собственная ЛР4, pinned dependencies,
[Valkey WAITAOF](https://valkey.io/commands/waitaof/),
[Valkey XAUTOCLAIM](https://valkey.io/commands/xautoclaim/),
[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html).
При проектировании, реализации, тестах и отчёте использована помощь ИИ.
Код/данные/история других студентов не использовались; результаты — реальные
локальные запуски, а remote CI отдельно не считается прошедшим.
Один host/LB/PG/S3/broker — SPOF, AOF/fsync/volume не заменяет backup и защиту
от потери диска. HTTP допустим для локальных синтетических данных; удалённый
доступ требует TLS. Строгого production exactly-once нет. HEAD metadata не
заменяет полный periodic checksum audit. Удалённые bytes не восстановить по SHA.
Весь inventory GC хранится в RAM, outbox/events/audit требуют retention при
долгой эксплуатации. Нет global admission quota: при долгой недоступности
worker/broker PG outbox может расти; исследование перегрузки — другой трек.
Short synthetic fixtures и повторы ограничивают обобщение на большие файлы.
Успешно завершившиеся child samples не включают RSS убитых процессов.
## Настройки
Compose использует PUBLIC_PORT, API_REPLICAS/CPUS/MEMORY, DB_POOL_SIZE,
SESSION_TTL_SECONDS, WORKER_CONCURRENCY, JOB_MAX_ATTEMPTS/TIMEOUT/LEASE и
STAGING_RETENTION_SECONDS. Значения .env.example для S3 endpoint/region/bucket,
MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и idempotency TTL документируют фиксированный
контракт реализации: s3:9000/us-east-1/messenger,10MiB/40M/24h. Их нельзя изменить
одним редактированием env: соответствующие constants/SQL надо менять согласованно.
Secret *_FILE paths заданы в compose; значения создаёт bootstrap.
## Карта критериев и история проверок
| Уровень / требование | Реализация / автоматическая проверка |
| --- | --- |
| 3: минимум ЛР4 +202queued | API/ACL/byte/pixel/MIME guards;112 HTTP tests, public smoke |
| 3: durable broker и отдельный worker | persistent Valkey Streams/AOF/WAITAOF; broker SIGKILL с volume |
| 3:4states, original/thumbnail, retry | src/worker.py, schema-valid metadata; failed/retry scenarios |
| 3:worker stop→queued link→restart→ready | commit_before_publish_and_worker_stopped |
| 4:crop/optimized/metadata removal | HTTP geometry/EXIF/alpha +22 unit tests |
| 4:outbox gap и publisher crash | API COMMIT-before-publish и relay XADD-before-PG-marker |
| 4:finite backoff/permanent/transient/replay |5attempt exhaustion, timeout, failed registry, audit, ready replay rejection |
| 4:crash после PUT доACK, lease, bounded slots | before_effect/after_effect/after_commit ×2; live state/actual limits подтверждены clean-clone-final |
| 4:cleanup после фиксации/retention | verify_cleanup seed/verify, реальныйS3возраст, dry/apply/repeat |
| 5:versioned idempotency track | duplicates/reorder и held stalev1→v2×2, no old publication/hash change |
[evidence/http-first.json](evidence/http-first.json):112 HTTP PASS404.42s,
source/runtime bb4e8a2. Отдельные22unit PASS после добавления real pixel,
checksum и malformed image worker tests. Public smoke PASS.
Первые полные matrix runs сохранены честно как незавершённые:
[pipeline-first-failed.json](evidence/pipeline-first-failed.json) —2cases,
затем тест сравнил очередь при работающем relay; snapshot изолирован остановкой
relay, повтор подтвердил сохранность exact entries после broker SIGKILL.
[pipeline-second-failed.json](evidence/pipeline-second-failed.json) —15cases
прошли, затем отдельные controller invocations съели12s бюджет старого child.
V2 была ready, но parent успел убитьv1; это ещё не доказательство CAS.
Новая координация использует один заранее запущенный controller: ждёт phase,
создаётv2, ждёт ready, отпускаетv1 и видит stale_completion_rejected внутри
живого окна. [pipeline-versions-first.json](evidence/pipeline-versions-first.json)
— **scope=versions**,4cases PASS:2version races, malformed notification,
private/IAM/otherAPI. Это целевой повтор, а не19cases полного matrix.
Приложение в этих исправлениях harness не менялось. Финальный clean clone
завершил scope=all, включая строгие ACK/live-state/resource assertions.
[evidence/gc-first.json](evidence/gc-first.json): реальный возраст orphan
1279.103s (выше300s); dry-run сохранил bytes, apply удалил expired objects,
повтор не изменил ничего. Active/current/fresh и неистёкший reservation
защищены; source_expired replay отвергнут; expired reservation больше не
проходит handoff predicate. Fixture metadata failed состарены SQL отдельно,
но возраст S3 LastModified не подменялся. Cleanup fixtures синтетические и
не обещают восстановление bytes из checksum.
Для финальной проверки worker перед fault hook сохраняет diagnostic event
worker_resource_limits с фактическим RLIMIT_AS и cgroup memory.max/cpu.max.
Сбор показаний самим child исключает влияние медленного внешнего DockerCLI
на12s test budget. Это диагностическое добавление также проверяется новым
образом из чистого clone; предыдущие HTTP/source evidence не переименованы.
Первый cluster receipt [cluster-first-failed.json](evidence/cluster-first-failed.json)
сохраняет успешные отказы двухAPI. Rolling занял дольше45s окна, поэтому
генератор завершился сразу с action и не получил post-action observation.
Verifier теперь обязательно наблюдает ещё2s после action, сохраняя все
проверки данных и failover≤10s. Это исправление наблюдения, не ослабление SLA.
[evidence/cluster-first.json](evidence/cluster-first.json): complete PASS;
API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762.
Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены.
Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились,
прежний token и file сохранились. Это не zero-error rolling.
## Уточнение строгого окна COMMIT→ACK
Первый чистый повтор [clean-clone-first.json](evidence/clean-clone-first.json)
sourceeb6d229 прошёл112HTTP379.34s/22unit/19cases/cluster/cleanup/hashes.
Однако в after_commit snapshot уже был duplicate_decision: второйconsumer
XAUTOCLAIM после3s успевал ACK раньшеSIGKILL. Это безопасно для данных,
но не доказывает конкретное окно COMMIT-before-ACK. Первый receipt сохранён
как результат прежней версии проверок; строгая граница требует нового повтора.
Reclaim idle теперь равен lease(23s normal/15s test), превышая child timeout
20s/12s. Это предотвращает ранний захват чужого live entry. Новый тест
снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и
отсутствие child_finished; после restart требует ready duplicate decision
и исчезновение notifications. Scope=all, данные и все прежние assertions
сохранены. Финальное доказательство — clean-clone-final.json (PASS).
## Финальный результат из чистого clone
Дата запуска UTC: 2026-11-15T20:48:55.825623+00:00. Source: `442163b0f0a549ef065cb13ee39b5ca2f8580c18`.
Последующие изменения отчёта/evidence не меняют исполняемые исходники.
Host: Apple M1, 8CPU, 8GiB RAM; Docker VM около6GiB RAM. Лимиты
контейнеров перечислены выше и сохранены в cluster runtime snapshots.
Новые secrets/env/volumes, отдельный image/project/port; после проверки
удалён только временный проект. Основные данные сохранены.
HTTP: `112 passed in 397.50s (0:06:37)`. Unit:22PASS; public smoke,
check/validate/lint, git fsck и сверка SHA каждого src/*.py с OCI revision
всех6контейнеров (2API/worker/relay/reconciler/controller) — PASS.
| Сценарий | Итоговые jobs: version/state/attempt/cycle/fence | Publications | Rows/keys |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| staging_before_handoff_crash | см. raw assertions | — | — |
| commit_before_publish_and_worker_stopped | v1/ready/1/1/1 | 1 | 4/4 |
| confirmed_broker_entry_survives_sigkill | v1/ready/1/1/1 | 1 | 4/4 |
| relay_crash_after_xadd_before_pg_marker | v1/ready/1/1/1 | 1 | 4/4 |
| before_effect_sigkill_repeat_1 | v1/ready/2/1/3 | 1 | 4/4 |
| after_effect_sigkill_repeat_1 | v1/ready/2/1/3 | 1 | 4/4 |
| after_commit_sigkill_repeat_1 | v1/ready/1/1/1 | 1 | 4/4 |
| before_effect_sigkill_repeat_2 | v1/ready/2/1/3 | 1 | 4/4 |
| after_effect_sigkill_repeat_2 | v1/ready/2/1/3 | 1 | 4/4 |
| after_commit_sigkill_repeat_2 | v1/ready/1/1/1 | 1 | 4/4 |
| transient_retry_with_jitter | v1/ready/2/1/3 | 1 | 4/4 |
| hung_child_killed_and_retried | v1/ready/2/1/3 | 1 | 4/4 |
| permanent_failed_registry_and_audited_replay | v1/ready/1/2/4 | 1 | 4/4 |
| finite_retry_exhaustion_then_explicit_replay | v1/ready/1/2/12 | 1 | 4/4 |
| reordered_and_duplicate_notifications | v1/ready/1/1/1; v1/ready/1/1/1 | 1; 1 | 4/4; 4/4 |
| stale_v1_after_v2_repeat_1 | v1/obsolete/1/1/2; v2/ready/1/1/1 | 0; 1 | 0/4; 4/4 |
| stale_v1_after_v2_repeat_2 | v1/obsolete/1/1/2; v2/ready/1/1/1 | 0; 1 | 0/4; 4/4 |
| malformed_notification_rejected | см. raw assertions | — | — |
| private_iam_and_other_api_download | см. raw assertions | — | — |
## Ограничения и следующие шаги
Полные UUID/attempts/fences/events/checksums — в
[pipeline-final.json](evidence/pipeline-final.json) и
[effects.csv](evidence/pipeline-analysis-final/effects.csv).
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Строгие COMMIT→ACK snapshots перед SIGKILL:
- `after_commit_sigkill_repeat_1`: job `efd4aa67-b47e-462f-8e86-614c5dbb8a17`, pending entries `1794776649599-0`; ready/publication1, child_finished отсутствует.
- `after_commit_sigkill_repeat_2`: job `c422c6de-4c05-43ea-b748-e4ff856a70c8`, pending entries `1794776766791-0`; ready/publication1, child_finished отсутствует.
После restart оба сценария дождались duplicate_decision(state=ready) и
отсутствия job notifications в stream (ACK+XDEL), без нового resize.
Два held stalev1 действительно дошли до stale_completion_rejected после
readyv2; уv1 ноль publications, hashes всех4currentv2 не изменились.
Агрегация: 17 ready observations — каждое с1publication,4rows/4keys;
2 obsolete observations — с0publications. Это конечные наблюдения
синтетической матрицы, а не формальное доказательство для всех возможных расписаний.
![Measured pipeline invariants and resource samples](evidence/pipeline-analysis-final/pipeline-results.png)
15 successful child samples: callback elapsed 0.2563–0.4592s,
CPU 0.2274–0.3770s, peakRSS 79.12–79.59MiB.
Elapsed/CPU измеряют тело callback и не включают spawn/import до него; RSS
учитывает startup процесса. Убитые child не входят в successful samples.
Это не queue lag, p95 до ready или throughput/SLA. Фактические лимиты
наблюдал сам child: RLIMIT_AS soft/hard805306368bytes, cgroup memory.max
1610612736bytes; CPU quota — в observed_limits raw case.
Cluster failovers (confirmed writes / transient errors / recovery seconds):
- 202 / 10 / 4.776s; confirmed data preserved, duplicate texts отсутствуют.
- 218 / 8 / 2.971s; confirmed data preserved, duplicate texts отсутствуют.
Rolling: 739 confirmed writes, 45 transient errors;
исходные API CID заменены, replay и данные сохранены. Public failover и
persistence с прежними token/file прошли; StartedAt хранилищ изменился.
Кратковременные ошибки учитываются, zero-error rolling не заявлен.
Cleanup: реальный возраст S3 orphan 754.765s;
dry-run 10actions без изменения bytes; apply 10actions;
repeat0actions. Active/current/fresh и действующий reservation защищены,
expired source replay отвергнут, expired writer fenced. Возраст fixture
metadata менялся SQL отдельно; S3 LastModified не подменялся.
Raw: [cleanup-final.json](evidence/cleanup-final.json).
## Публикация
Публичный fork: [wh1tet3a/mtusi2026lab5](https://gitea.privcloud.tech/wh1tet3a/mtusi2026lab5).
PR: [ЛР 5 — БВТ2301 — Маслов — на 5, №70](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/70),
ветка `solution/bvt2301-maslov` → преподавательская `main`.
При публикации remote CI имел pending/0checks; локальный clean PASS не выдаётся
за результат удалённого CI. Commit со ссылкой на PR меняет только документацию.
+48
View File
@@ -0,0 +1,48 @@
# PostgreSQL / S3 / broker — ЛР5
```mermaid
flowchart LR
C[Client] --> LB[HAProxy]
LB --> A[API 1/2]
A --> S[Private S3 staging]
A --> P[(PG attachment + job + outbox)]
P --> R[Relay + watchdog]
R --> B[(Valkey Streams + AOF)]
B --> W[Worker 2 bounded child slots]
W --> V[S3 versioned original + 3 variants]
W --> E[(PG fenced atomic publication)]
E --> A
```
Core: users→sessions; chats↔members; chats→messages; message_attachments
имеет составные FK chat/message и chat/attachment. Сессии и idempotency
сохраняются в PG; cache не является источником истины. Row locks и unique
(user,method,path,key) сохраняют Message+result атомарно.
Attachment хранит свойства original, target pipeline_version и
published_version. Jobs unique(attachment_id,pipeline_version), UUID logical ID,
state,attempt/cycle,fence,lease/owner,next_at,safeerror. Outbox содержит только
schema-valid envelope, due/published. Consumer group хранит pending entries,
но окончательные решения находятся в PG. Failed registry — jobs.state=failed.
Результаты: attachment_variants PK(attachment_id,pipeline_version,name),
unique object_key; effect_publications PK(attachment_id,pipeline_version).
Worker в одной транзакции проверяет target version, fence, live lease и
processing, записывает4variants+effect_publication+ready. Object bytes
предшествуют COMMIT; дубли могут повторить PUT тех же bytes/keys.
Публикацию новой версии нельзя перезаписать старым fence/version.
Upload reservation120s предотвращает поздний COMMIT после GC: row lock плюс
проверка expiry нужны и при handoff, и при удалении просроченного reservation.
Unconfirmed bytes не становятся accepted jobs сами по себе.
S3: staging `objects/staging/<UUID>/original`, final
`objects/<attachment>/v<version>/{original,thumbnail,crop,optimized}`.
Filename — только отображаемое поле, не путь. IAM ограничен messenger/objects/*,
корневые credentials получает только S3/bootstrap. PG содержит metadata/SHA,
а не bytes. Broker envelope содержит ID/version/attempt/time, не storage paths.
pipeline_audit сохраняет explicit operator replay/version; pipeline_events —
наблюдаемые решения без secrets. pipeline_faults — test-only single-use hooks,
выключены без PIPELINE_FAULTS=1. В production API не имеет broker credentials;
test controller появляется только в compose.test.yaml и не публикует портов.
+30
View File
@@ -0,0 +1,30 @@
# Explicit test-only fault instrumentation, never enabled by default.
services:
api:
environment:
PIPELINE_FAULTS: "1"
relay:
environment:
PIPELINE_FAULTS: "1"
worker:
environment:
PIPELINE_FAULTS: "1"
JOB_TIMEOUT_SECONDS: "12"
JOB_LEASE_SECONDS: "15"
controller:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -c, "import time; time.sleep(86400)"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
cpus: 0.5
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
+219
View File
@@ -0,0 +1,219 @@
name: mtusi-lab5-maslov
services:
postgres:
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
PGDATA: /var/lib/postgresql/data/pgdata
secrets: [postgres_password]
volumes:
- postgres_data:/var/lib/postgresql
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
cpus: 1
mem_limit: 512m
memswap_limit: 512m
cache:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/valkey.conf]
secrets:
- cache_password
- source: valkey_config
target: valkey.conf
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping']
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
memswap_limit: 128m
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres: {condition: service_healthy}
cpus: 0.5
mem_limit: 256m
read_only: true
api:
image: ${API_IMAGE:-mtusi-messenger:lab5}
build:
context: .
args:
VCS_REF: ${VCS_REF:-unknown}
scale: ${API_REPLICAS:-2}
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
secrets: [database_url, cache_url, s3_access_key, s3_secret_key]
depends_on:
migrate: {condition: service_completed_successfully}
cache: {condition: service_healthy}
s3-bootstrap: {condition: service_completed_successfully}
cpus: ${API_CPUS:-0.5}
mem_limit: ${API_MEMORY:-384m}
memswap_limit: ${API_MEMORY:-384m}
pids_limit: 128
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=128m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
stop_grace_period: 10s
s3:
image: ${S3_IMAGE:-mtusi-storage:minio-20251015}
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
cpus: 1
mem_limit: 1024m
memswap_limit: 1024m
read_only: true
tmpfs: ["/tmp:rw,nosuid,size=32m"]
environment:
HOME: /tmp
security_opt: [no-new-privileges:true]
s3-bootstrap:
image: ${MC_IMAGE:-mtusi-storage:mc-20250813}
build: {context: ., dockerfile: infra/s3.Dockerfile, target: bootstrap}
secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key]
depends_on:
s3: {condition: service_healthy}
cpus: 0.25
mem_limit: 128m
read_only: true
tmpfs: ["/tmp:rw,nosuid,size=16m"]
reconciler:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.reconcile, --apply, --loop]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets: [database_url, s3_access_key, s3_secret_key]
depends_on:
migrate: {condition: service_completed_successfully}
s3-bootstrap: {condition: service_completed_successfully}
cpus: 0.25
mem_limit: 256m
memswap_limit: 256m
read_only: true
healthcheck: {disable: true}
broker:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command: [valkey-server, /run/secrets/broker.conf]
secrets:
- broker_password
- source: broker_config
target: broker.conf
volumes: [broker_data:/data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping']
interval: 2s
timeout: 2s
retries: 20
cpus: 0.5
mem_limit: 256m
memswap_limit: 256m
relay:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.pipeline]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
secrets: [database_url, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
cpus: 0.25
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
worker:
image: ${API_IMAGE:-mtusi-messenger:lab5}
pull_policy: never
command: [python, -m, src.worker]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23}
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_healthy}
s3-bootstrap: {condition: service_completed_successfully}
cpus: 2
mem_limit: 1536m
memswap_limit: 1536m
pids_limit: 128
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"]
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
stop_grace_period: 25s
lb:
image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9
ports:
- "127.0.0.1:${PUBLIC_PORT:-8085}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api: {condition: service_healthy}
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
read_only: true
tmpfs: ["/tmp:rw,nosuid,size=4m"]
security_opt: [no-new-privileges:true]
stop_grace_period: 10s
secrets:
broker_password: {file: ./secrets/broker_password}
broker_url: {file: ./secrets/broker_url}
broker_config: {file: ./secrets/broker.conf}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key: {file: ./secrets/s3_access_key}
s3_secret_key: {file: ./secrets/s3_secret_key}
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_password:
file: ./secrets/cache_password
cache_url:
file: ./secrets/cache_url
valkey_config:
file: ./secrets/valkey.conf
name: valkey.conf
volumes:
broker_data:
s3_data:
postgres_data:
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+159
View File
@@ -0,0 +1,159 @@
{
"source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18",
"seed": "clean-gc-seed.json",
"complete": true,
"real_s3_age_seconds": 754.765394,
"dry_run": {
"apply": false,
"retention_seconds": 300,
"actions": [
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original",
"applied": false
}
]
},
"apply": {
"apply": true,
"retention_seconds": 300,
"actions": [
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original",
"applied": true
}
]
},
"repeat": {
"apply": true,
"retention_seconds": 300,
"actions": []
},
"before": {
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": true,
"objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": true,
"objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": true,
"objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true,
"objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true,
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true
},
"after": {
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": false,
"objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": false,
"objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": false,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": false,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": false,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": false,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": false,
"objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true,
"objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true,
"objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true,
"objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true
},
"dry_run_unchanged": true,
"active_current_and_fresh_protected": true,
"expired_source_replay_refused": true,
"expired_writer_fenced": true
}
File diff suppressed because it is too large Load Diff
+20
View File
@@ -0,0 +1,20 @@
Traceback (most recent call last):
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "<repository>/scripts/verify_cluster.py", line 342, in <module>
main()
~~~~^^
File "<repository>/scripts/verify_cluster.py", line 278, in main
report["rolling"] = exercise(
~~~~~~~~^
client,
^^^^^^^
...<3 lines>...
seconds=45,
^^^^^^^^^^^
)
^
File "<repository>/scripts/verify_cluster.py", line 114, in exercise
assert recovery is not None and (seconds > 20 or recovery <= 10)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
File diff suppressed because it is too large Load Diff
+209
View File
@@ -0,0 +1,209 @@
{
"source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88",
"seed": "gc-seed-first.json",
"complete": true,
"real_s3_age_seconds": 1279.102705,
"dry_run": {
"apply": false,
"retention_seconds": 300,
"actions": [
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original",
"applied": false
},
{
"action": "delete_expired_object",
"key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original",
"applied": false
}
]
},
"apply": {
"apply": true,
"retention_seconds": 300,
"actions": [
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original",
"applied": true
},
{
"action": "delete_expired_object",
"key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original",
"applied": true
}
]
},
"repeat": {
"apply": true,
"retention_seconds": 300,
"actions": []
},
"before": {
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": true,
"objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": true,
"objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": true,
"objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true,
"objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true,
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true
},
"after": {
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": false,
"objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": false,
"objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": false,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": false,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": false,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": false,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": false,
"objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true,
"objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true,
"objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true,
"objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true
},
"dry_run_unchanged": true,
"active_current_and_fresh_protected": true,
"expired_source_replay_refused": true,
"expired_writer_fenced": true
}
+20
View File
@@ -0,0 +1,20 @@
{
"seed_id": "ae4b883f7b784f40ae8f5e1c78ac3091",
"source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2",
"date": "2026-11-15T19:46:23.318659+00:00",
"ready_attachment": "d6f10e70-74e9-4394-be76-5c0b14d3f91f",
"ready_job": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35",
"chat": "3428fad7-2178-4730-b56a-4b579bdd4966",
"owner": "36007eaa-ceba-4f80-98fa-13c912f73873",
"version2_job": "cc18c75e-8886-4333-bae8-5da93d626c73",
"orphan": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan",
"failed_id": "b98bc768-f36a-4615-9097-c93a78bfed05",
"active_id": "69150761-a8a1-437b-91b0-b363d7a1507a",
"reservation_id": "ec7a6893-8d3b-4488-b0cd-06e53499cf9d",
"expired_reservation_id": "402b3515-91ad-40e0-8e36-6627feb18be7",
"failed_source": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original",
"active_source": "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original",
"reservation_key": "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original",
"expired_reservation_key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original",
"seeded_unix": 1794771995.922627
}
+12
View File
@@ -0,0 +1,12 @@
{
"source_commit": "bb4e8a2",
"runtime_revision": "bb4e8a2",
"public_smoke": "PASS",
"http_tests": 112,
"passed": 112,
"failed": 0,
"seconds": 404.42,
"unit_tests": 19,
"unit_result": "PASS",
"output": "........................................................................ [ 64%]\n........................................ [100%]\n112 passed in 404.42s (0:06:44)\n"
}
+1
View File
@@ -0,0 +1 @@
{"result":"INVALID_ENVIRONMENT","cause":"Inherited student fixture default pointed to stopped lab4 port8084, not lab5 port8085; all connections refused; no API correctness conclusion.","failed":40,"errors":72,"seconds":8.77}
@@ -0,0 +1,20 @@
case,role,job_id,version,attempt,cycle,fence,state,publications,variant_rows,physical_keys
commit_before_publish_and_worker_stopped,effect,3e2eccd9-713d-4982-a025-5537b66c114c,1,1,1,1,ready,1,4,4
confirmed_broker_entry_survives_sigkill,effect,5747edc1-9de6-40ba-a143-85ded56744d1,1,1,1,1,ready,1,4,4
relay_crash_after_xadd_before_pg_marker,effect,2177d978-9a49-4856-a9e9-6211e94e9246,1,1,1,1,ready,1,4,4
before_effect_sigkill_repeat_1,effect,3f046f7c-825f-446f-bf52-07e4cd7e330b,1,2,1,3,ready,1,4,4
after_effect_sigkill_repeat_1,effect,64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0,1,2,1,3,ready,1,4,4
after_commit_sigkill_repeat_1,effect,efd4aa67-b47e-462f-8e86-614c5dbb8a17,1,1,1,1,ready,1,4,4
before_effect_sigkill_repeat_2,effect,6eacee3c-6b6b-4435-bd05-bb50c29df24f,1,2,1,3,ready,1,4,4
after_effect_sigkill_repeat_2,effect,0d40fa0a-e6b9-49cb-907d-e82faa346906,1,2,1,3,ready,1,4,4
after_commit_sigkill_repeat_2,effect,c422c6de-4c05-43ea-b748-e4ff856a70c8,1,1,1,1,ready,1,4,4
transient_retry_with_jitter,effect,00893137-7e67-434b-8764-5c85ed5353d2,1,2,1,3,ready,1,4,4
hung_child_killed_and_retried,effect,eefeceb7-6e35-4358-bbac-772170b964fa,1,2,1,3,ready,1,4,4
permanent_failed_registry_and_audited_replay,effect,88929609-e34a-4b5f-ad36-ffce561ebde8,1,1,2,4,ready,1,4,4
finite_retry_exhaustion_then_explicit_replay,effect,b6e323c3-3e4a-45fd-a4a5-709094e97485,1,1,2,12,ready,1,4,4
reordered_and_duplicate_notifications,1,e7f4c924-cb49-40a5-b15b-b5bd8b3bd401,1,1,1,1,ready,1,4,4
reordered_and_duplicate_notifications,2,108ade45-7571-4362-92d5-bde3c6232fe9,1,1,1,1,ready,1,4,4
stale_v1_after_v2_repeat_1,v1,bc562c08-8221-4f9d-b5ff-890d16987495,1,1,1,2,obsolete,0,0,4
stale_v1_after_v2_repeat_1,v2,75ceffdc-919b-4f92-96c4-2e0c775850d7,2,1,1,1,ready,1,4,4
stale_v1_after_v2_repeat_2,v1,61e16816-661d-46bd-882e-80cc1cab86dd,1,1,1,2,obsolete,0,0,4
stale_v1_after_v2_repeat_2,v2,6d569c1e-c44c-4b65-a49d-ae731029270a,2,1,1,1,ready,1,4,4
1 case role job_id version attempt cycle fence state publications variant_rows physical_keys
2 commit_before_publish_and_worker_stopped effect 3e2eccd9-713d-4982-a025-5537b66c114c 1 1 1 1 ready 1 4 4
3 confirmed_broker_entry_survives_sigkill effect 5747edc1-9de6-40ba-a143-85ded56744d1 1 1 1 1 ready 1 4 4
4 relay_crash_after_xadd_before_pg_marker effect 2177d978-9a49-4856-a9e9-6211e94e9246 1 1 1 1 ready 1 4 4
5 before_effect_sigkill_repeat_1 effect 3f046f7c-825f-446f-bf52-07e4cd7e330b 1 2 1 3 ready 1 4 4
6 after_effect_sigkill_repeat_1 effect 64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0 1 2 1 3 ready 1 4 4
7 after_commit_sigkill_repeat_1 effect efd4aa67-b47e-462f-8e86-614c5dbb8a17 1 1 1 1 ready 1 4 4
8 before_effect_sigkill_repeat_2 effect 6eacee3c-6b6b-4435-bd05-bb50c29df24f 1 2 1 3 ready 1 4 4
9 after_effect_sigkill_repeat_2 effect 0d40fa0a-e6b9-49cb-907d-e82faa346906 1 2 1 3 ready 1 4 4
10 after_commit_sigkill_repeat_2 effect c422c6de-4c05-43ea-b748-e4ff856a70c8 1 1 1 1 ready 1 4 4
11 transient_retry_with_jitter effect 00893137-7e67-434b-8764-5c85ed5353d2 1 2 1 3 ready 1 4 4
12 hung_child_killed_and_retried effect eefeceb7-6e35-4358-bbac-772170b964fa 1 2 1 3 ready 1 4 4
13 permanent_failed_registry_and_audited_replay effect 88929609-e34a-4b5f-ad36-ffce561ebde8 1 1 2 4 ready 1 4 4
14 finite_retry_exhaustion_then_explicit_replay effect b6e323c3-3e4a-45fd-a4a5-709094e97485 1 1 2 12 ready 1 4 4
15 reordered_and_duplicate_notifications 1 e7f4c924-cb49-40a5-b15b-b5bd8b3bd401 1 1 1 1 ready 1 4 4
16 reordered_and_duplicate_notifications 2 108ade45-7571-4362-92d5-bde3c6232fe9 1 1 1 1 ready 1 4 4
17 stale_v1_after_v2_repeat_1 v1 bc562c08-8221-4f9d-b5ff-890d16987495 1 1 1 2 obsolete 0 0 4
18 stale_v1_after_v2_repeat_1 v2 75ceffdc-919b-4f92-96c4-2e0c775850d7 2 1 1 1 ready 1 4 4
19 stale_v1_after_v2_repeat_2 v1 61e16816-661d-46bd-882e-80cc1cab86dd 1 1 1 2 obsolete 0 0 4
20 stale_v1_after_v2_repeat_2 v2 6d569c1e-c44c-4b65-a49d-ae731029270a 2 1 1 1 ready 1 4 4
Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

@@ -0,0 +1,22 @@
{
"source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18",
"cases": 19,
"observed_ready_effects": 17,
"observed_obsolete_effects": 2,
"all_ready_have_one_publication_and_four_variants": true,
"all_obsolete_have_zero_publications": true,
"successful_child_samples": 15,
"child_elapsed_range_seconds": [
0.2563110829996731,
0.4591874589996223
],
"child_cpu_range_seconds": [
0.22742345900000016,
0.37699009000000006
],
"child_peak_rss_range_mib": [
79.125,
79.5859375
],
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned"
}
File diff suppressed because it is too large Load Diff
+109
View File
@@ -0,0 +1,109 @@
{
"source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2",
"date": "2026-11-15T19:46:36.154413+00:00",
"track": 2,
"complete": false,
"cases": [
{
"case": "staging_before_handoff_crash",
"no_202": true,
"no_attachment_or_job_created": true,
"orphan_retained_for_cleanup": true
},
{
"case": "commit_before_publish_and_worker_stopped",
"attachment": "f554e320-4881-4087-b813-9f28607fc3b1",
"effect": {
"job": {
"id": "1016e737-7ec1-4fde-9946-a342d70b2a71",
"attachment_id": "f554e320-4881-4087-b813-9f28607fc3b1",
"pipeline_version": 1,
"source_key": "objects/staging/f554e320-4881-4087-b813-9f28607fc3b1/original",
"state": "ready",
"attempt": 1,
"cycle": 1,
"fence": 1,
"lease_owner": null,
"lease_until": null,
"next_at": "2026-11-15 19:47:10.665115+00:00",
"error_code": null,
"created_at": "2026-11-15 19:47:10.665115+00:00",
"updated_at": "2026-11-15 19:47:36.689673+00:00"
},
"variants": [
{
"name": "crop",
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop",
"sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef",
"size_bytes": 1077
},
{
"name": "optimized",
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized",
"sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8",
"size_bytes": 194
},
{
"name": "original",
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original",
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df",
"size_bytes": 742
},
{
"name": "thumbnail",
"object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail",
"sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199",
"size_bytes": 1749
}
],
"publications": 1,
"physical_keys": [
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop",
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized",
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original",
"objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail"
],
"events": [
{
"event": "claimed",
"detail": {
"fence": 1,
"attempt": 1,
"version": 1
},
"created_at": "2026-11-15 19:47:34.529681+00:00"
},
{
"event": "published",
"detail": {
"entry": "1794772054509-0",
"outbox": 11
},
"created_at": "2026-11-15 19:47:34.584052+00:00"
},
{
"event": "effect_committed",
"detail": {
"fence": 1,
"version": 1,
"variants": 4
},
"created_at": "2026-11-15 19:47:36.690043+00:00"
},
{
"event": "child_finished",
"detail": {
"result": "ready",
"seconds": 0.2740797910000765,
"cpu_seconds": 0.2354738329999997,
"rss_peak_mib": 79.3125
},
"created_at": "2026-11-15 19:47:36.847178+00:00"
}
]
},
"queued_link_allowed": true,
"api_and_broker_restarted": true
}
]
}
+12
View File
@@ -0,0 +1,12 @@
PASS: staging_before_handoff_crash
PASS: commit_before_publish_and_worker_stopped
Traceback (most recent call last):
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "<repository>/scripts/verify_pipeline.py", line 602, in <module>
main()
~~~~^^
File "<repository>/scripts/verify_pipeline.py", line 318, in main
assert restored == ids
^^^^^^^^^^^^^^^
AssertionError
File diff suppressed because it is too large Load Diff
+30
View File
@@ -0,0 +1,30 @@
PASS: staging_before_handoff_crash
PASS: commit_before_publish_and_worker_stopped
PASS: confirmed_broker_entry_survives_sigkill
PASS: relay_crash_after_xadd_before_pg_marker
PASS: before_effect_sigkill_repeat_1
PASS: after_effect_sigkill_repeat_1
PASS: after_commit_sigkill_repeat_1
PASS: before_effect_sigkill_repeat_2
PASS: after_effect_sigkill_repeat_2
PASS: after_commit_sigkill_repeat_2
PASS: transient_retry_with_jitter
PASS: hung_child_killed_and_retried
PASS: permanent_failed_registry_and_audited_replay
PASS: finite_retry_exhaustion_then_explicit_replay
PASS: reordered_and_duplicate_notifications
Traceback (most recent call last):
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "<repository>/scripts/verify_pipeline.py", line 604, in <module>
report["worker_metrics"] = inside(
File "<repository>/scripts/verify_pipeline.py", line 481, in main
"reordered_and_duplicate_notifications",
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<4 lines>...
},
^
File "<repository>/scripts/verify_pipeline.py", line 48, in wait
raise TimeoutError("Fault scenario condition not reached")
TimeoutError: Fault scenario condition not reached
+980
View File
@@ -0,0 +1,980 @@
{
"source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88",
"date": "2026-11-15T20:03:53.705513+00:00",
"track": 2,
"scope": "versions",
"complete": true,
"cases": [
{
"case": "stale_v1_after_v2_repeat_1",
"v1": {
"job": {
"id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46",
"attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd",
"pipeline_version": 1,
"source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original",
"state": "obsolete",
"attempt": 1,
"cycle": 1,
"fence": 2,
"lease_owner": null,
"lease_until": null,
"next_at": "2026-11-15 20:03:59.131799+00:00",
"error_code": null,
"created_at": "2026-11-15 20:03:59.131799+00:00",
"updated_at": "2026-11-15 20:04:25.401001+00:00"
},
"variants": [],
"publications": 0,
"physical_keys": [
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/crop",
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/optimized",
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/original",
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/thumbnail"
],
"events": [
{
"event": "published",
"detail": {
"entry": "1794773039190-0",
"outbox": 43
},
"created_at": "2026-11-15 20:03:59.200524+00:00"
},
{
"event": "published",
"detail": {
"entry": "1794773054245-0",
"outbox": 44
},
"created_at": "2026-11-15 20:04:14.251008+00:00"
},
{
"event": "version_controller_waiting",
"detail": {},
"created_at": "2026-11-15 20:04:18.270502+00:00"
},
{
"event": "claimed",
"detail": {
"fence": 1,
"attempt": 1,
"version": 1
},
"created_at": "2026-11-15 20:04:25.403107+00:00"
},
{
"event": "stale_completion_rejected",
"detail": {
"fence": 1,
"version": 1
},
"created_at": "2026-11-15 20:04:30.862701+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773039190-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:30.980982+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773054245-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:30.994675+00:00"
},
{
"event": "child_finished",
"detail": {
"result": "stale"
},
"created_at": "2026-11-15 20:04:31.083095+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079900-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.913386+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079902-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.914038+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079904-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.920903+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079905-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.922028+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079906-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.925707+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079908-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.926905+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079909-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.930906+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079910-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.931649+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079911-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.935325+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079912-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.935997+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079913-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.940184+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773079914-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:04:39.940743+00:00"
}
]
},
"v2": {
"job": {
"id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9",
"attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd",
"pipeline_version": 2,
"source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original",
"state": "ready",
"attempt": 1,
"cycle": 1,
"fence": 1,
"lease_owner": null,
"lease_until": null,
"next_at": "2026-11-15 20:04:28.320656+00:00",
"error_code": null,
"created_at": "2026-11-15 20:04:28.320656+00:00",
"updated_at": "2026-11-15 20:04:30.796918+00:00"
},
"variants": [
{
"name": "crop",
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop",
"sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef",
"size_bytes": 1077
},
{
"name": "optimized",
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized",
"sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced",
"size_bytes": 194
},
{
"name": "original",
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original",
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df",
"size_bytes": 742
},
{
"name": "thumbnail",
"object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail",
"sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199",
"size_bytes": 1749
}
],
"publications": 1,
"physical_keys": [
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop",
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized",
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original",
"objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail"
],
"events": [
{
"event": "new_version",
"detail": {
"previous_job": "51c2387d-8bc4-4f37-9ad9-b722464c1f46"
},
"created_at": "2026-11-15 20:04:28.323557+00:00"
},
{
"event": "published",
"detail": {
"entry": "1794773068363-0",
"outbox": 45
},
"created_at": "2026-11-15 20:04:28.367667+00:00"
},
{
"event": "claimed",
"detail": {
"fence": 1,
"attempt": 1,
"version": 2
},
"created_at": "2026-11-15 20:04:28.373092+00:00"
},
{
"event": "effect_committed",
"detail": {
"fence": 1,
"version": 2,
"variants": 4
},
"created_at": "2026-11-15 20:04:30.798523+00:00"
},
{
"event": "child_finished",
"detail": {
"result": "ready",
"seconds": 0.29709320799975103,
"cpu_seconds": 0.24198554100000003,
"rss_peak_mib": 79.61328125
},
"created_at": "2026-11-15 20:04:30.965510+00:00"
}
]
},
"new_hashes_unchanged": true
},
{
"case": "stale_v1_after_v2_repeat_2",
"v1": {
"job": {
"id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf",
"attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589",
"pipeline_version": 1,
"source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original",
"state": "obsolete",
"attempt": 1,
"cycle": 1,
"fence": 2,
"lease_owner": null,
"lease_until": null,
"next_at": "2026-11-15 20:05:03.917398+00:00",
"error_code": null,
"created_at": "2026-11-15 20:05:03.917399+00:00",
"updated_at": "2026-11-15 20:05:25.442589+00:00"
},
"variants": [],
"publications": 0,
"physical_keys": [
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/crop",
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/optimized",
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/original",
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/thumbnail"
],
"events": [
{
"event": "published",
"detail": {
"entry": "1794773103958-0",
"outbox": 46
},
"created_at": "2026-11-15 20:05:03.964545+00:00"
},
{
"event": "published",
"detail": {
"entry": "1794773119171-0",
"outbox": 47
},
"created_at": "2026-11-15 20:05:19.176271+00:00"
},
{
"event": "version_controller_waiting",
"detail": {},
"created_at": "2026-11-15 20:05:22.667065+00:00"
},
{
"event": "claimed",
"detail": {
"fence": 1,
"attempt": 1,
"version": 1
},
"created_at": "2026-11-15 20:05:25.444410+00:00"
},
{
"event": "stale_completion_rejected",
"detail": {
"fence": 1,
"version": 1
},
"created_at": "2026-11-15 20:05:30.563412+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773103958-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:30.764786+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773119171-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:30.775876+00:00"
},
{
"event": "child_finished",
"detail": {
"result": "stale"
},
"created_at": "2026-11-15 20:05:30.815598+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139795-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.811610+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139792-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.811671+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139797-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.818733+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139798-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.819185+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139799-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.824008+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139800-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.824598+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139801-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.829008+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139803-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.829676+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139804-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.833617+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139805-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.834470+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139805-1",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.838465+00:00"
},
{
"event": "duplicate_decision",
"detail": {
"entry": "1794773139806-0",
"state": "obsolete"
},
"created_at": "2026-11-15 20:05:39.839569+00:00"
}
]
},
"v2": {
"job": {
"id": "3c673acd-d8c5-44c7-8366-eed0480acdd8",
"attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589",
"pipeline_version": 2,
"source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original",
"state": "ready",
"attempt": 1,
"cycle": 1,
"fence": 1,
"lease_owner": null,
"lease_until": null,
"next_at": "2026-11-15 20:05:28.099769+00:00",
"error_code": null,
"created_at": "2026-11-15 20:05:28.099769+00:00",
"updated_at": "2026-11-15 20:05:30.544288+00:00"
},
"variants": [
{
"name": "crop",
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop",
"sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef",
"size_bytes": 1077
},
{
"name": "optimized",
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized",
"sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced",
"size_bytes": 194
},
{
"name": "original",
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original",
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df",
"size_bytes": 742
},
{
"name": "thumbnail",
"object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail",
"sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199",
"size_bytes": 1749
}
],
"publications": 1,
"physical_keys": [
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop",
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized",
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original",
"objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail"
],
"events": [
{
"event": "new_version",
"detail": {
"previous_job": "8679296c-7a76-46b4-a6a4-02b57d0dedaf"
},
"created_at": "2026-11-15 20:05:28.102749+00:00"
},
{
"event": "published",
"detail": {
"entry": "1794773128278-0",
"outbox": 48
},
"created_at": "2026-11-15 20:05:28.288692+00:00"
},
{
"event": "claimed",
"detail": {
"fence": 1,
"attempt": 1,
"version": 2
},
"created_at": "2026-11-15 20:05:28.307421+00:00"
},
{
"event": "effect_committed",
"detail": {
"fence": 1,
"version": 2,
"variants": 4
},
"created_at": "2026-11-15 20:05:30.544711+00:00"
},
{
"event": "child_finished",
"detail": {
"result": "ready",
"seconds": 0.28706112500003655,
"cpu_seconds": 0.24108328799999978,
"rss_peak_mib": 79.671875
},
"created_at": "2026-11-15 20:05:30.752666+00:00"
}
]
},
"new_hashes_unchanged": true
},
{
"case": "malformed_notification_rejected",
"durable_diagnostic": true
},
{
"case": "private_iam_and_other_api_download",
"security": {
"anonymous": 403,
"iam": {
"create_bucket": "AccessDenied",
"write_outside_prefix": "AccessDenied",
"list_other_bucket": "AccessDenied"
}
},
"attachment": "5875dfe8-09d9-4ded-9e31-a9329f0974ac",
"uploader": "c12dfe7a14a648809e07d709180d70e3",
"downloader": "0024aba8755f4f3d92a85c4dbb751f5a",
"sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df"
}
],
"worker_metrics": [
{
"job_id": "947fae60-fb2b-4000-ac82-76c0aa946f17",
"detail": {
"result": "ready",
"seconds": 0.5540742509999745,
"cpu_seconds": 0.47041774699999994,
"rss_peak_mib": 79.33203125
}
},
{
"job_id": "ac292276-9fc0-4d20-b9d0-1c784017f555",
"detail": {
"result": "ready",
"seconds": 0.26326079199998276,
"cpu_seconds": 0.23446212599999994,
"rss_peak_mib": 79.6796875
}
},
{
"job_id": "4d1ee9b7-32e0-44c4-8264-a831b9bc3bdb",
"detail": {
"result": "ready",
"seconds": 0.2615961250003238,
"cpu_seconds": 0.22782362500000009,
"rss_peak_mib": 79.44140625
}
},
{
"job_id": "1174230c-fb4f-4cdc-9db8-896d002f90a4",
"detail": {
"result": "ready",
"seconds": 0.2803810010000234,
"cpu_seconds": 0.24079599299999987,
"rss_peak_mib": 79.34765625
}
},
{
"job_id": "4b28f16f-162d-4f60-b1fd-3a907ee834e1",
"detail": {
"result": "ready",
"seconds": 0.2567913339998995,
"cpu_seconds": 0.228462414,
"rss_peak_mib": 79.47265625
}
},
{
"job_id": "12fa59f7-ed8a-4810-95cf-8b9be4c020e8",
"detail": {
"result": "ready",
"seconds": 0.28690104199995403,
"cpu_seconds": 0.23826428900000018,
"rss_peak_mib": 79.3046875
}
},
{
"job_id": "6247813c-df74-4eeb-9326-cbb13f860336",
"detail": {
"result": "ready",
"seconds": 0.4403786249999939,
"cpu_seconds": 0.4114635,
"rss_peak_mib": 108.4140625
}
},
{
"job_id": "9a3fad2f-b3cc-4a28-b996-10c84a4c4b94",
"detail": {
"result": "ready",
"seconds": 0.2648974580001777,
"cpu_seconds": 0.23192129000000028,
"rss_peak_mib": 79.3515625
}
},
{
"job_id": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35",
"detail": {
"result": "ready",
"seconds": 0.30638329200019143,
"cpu_seconds": 0.27590770600000014,
"rss_peak_mib": 79.4296875
}
},
{
"job_id": "cc18c75e-8886-4333-bae8-5da93d626c73",
"detail": {
"result": "ready",
"seconds": 0.2798381249999693,
"cpu_seconds": 0.24821178899999996,
"rss_peak_mib": 79.6640625
}
},
{
"job_id": "1016e737-7ec1-4fde-9946-a342d70b2a71",
"detail": {
"result": "ready",
"seconds": 0.2740797910000765,
"cpu_seconds": 0.2354738329999997,
"rss_peak_mib": 79.3125
}
},
{
"job_id": "6e004e09-d91d-42b9-907b-9fda074afeb3",
"detail": {
"result": "ready",
"seconds": 0.2604502920003142,
"cpu_seconds": 0.22423492300000003,
"rss_peak_mib": 79.328125
}
},
{
"job_id": "817ac0d3-2ae3-4721-a1c4-0d517f624f3a",
"detail": {
"result": "ready",
"seconds": 0.2753011250001691,
"cpu_seconds": 0.24031520900000003,
"rss_peak_mib": 79.31640625
}
},
{
"job_id": "a68327bb-1d99-4ca1-a541-c197ad0683ea",
"detail": {
"result": "ready",
"seconds": 0.2575967509997099,
"cpu_seconds": 0.23045399499999997,
"rss_peak_mib": 79.84765625
}
},
{
"job_id": "2b40b034-2ea5-4612-8d4e-f099e1e3f65d",
"detail": {
"result": "ready",
"seconds": 0.2825338749998991,
"cpu_seconds": 0.24960387800000006,
"rss_peak_mib": 79.30859375
}
},
{
"job_id": "681235f5-558b-44b8-af12-e61d2f6c65bf",
"detail": {
"result": "ready",
"seconds": 0.2647618330001933,
"cpu_seconds": 0.23189429199999978,
"rss_peak_mib": 79.6796875
}
},
{
"job_id": "e4b36262-0688-4b3c-90ce-9252e9633353",
"detail": {
"result": "ready",
"seconds": 0.2914882079999188,
"cpu_seconds": 0.24316999600000022,
"rss_peak_mib": 79.80078125
}
},
{
"job_id": "e1ebdd4b-996f-41d2-aa54-246cc9f4c8d3",
"detail": {
"result": "ready",
"seconds": 0.259311875000094,
"cpu_seconds": 0.23085762499999984,
"rss_peak_mib": 79.45703125
}
},
{
"job_id": "fbedc7f8-97a6-4c59-848c-d4ab8045d4d2",
"detail": {
"result": "ready",
"seconds": 0.26234775000011723,
"cpu_seconds": 0.22806161800000013,
"rss_peak_mib": 79.328125
}
},
{
"job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d",
"detail": {
"code": "processing_unavailable",
"result": "transient"
}
},
{
"job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d",
"detail": {
"result": "ready",
"seconds": 0.30820225099978416,
"cpu_seconds": 0.27197879400000025,
"rss_peak_mib": 79.34765625
}
},
{
"job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb",
"detail": {
"code": "worker_timeout",
"result": "transient"
}
},
{
"job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb",
"detail": {
"result": "ready",
"seconds": 0.26163899999983187,
"cpu_seconds": 0.22942266700000014,
"rss_peak_mib": 79.33203125
}
},
{
"job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1",
"detail": {
"code": "invalid_image",
"result": "permanent"
}
},
{
"job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1",
"detail": {
"result": "ready",
"seconds": 0.27349454200020773,
"cpu_seconds": 0.23941921199999983,
"rss_peak_mib": 79.28125
}
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"detail": {
"code": "processing_unavailable",
"result": "transient"
}
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"detail": {
"code": "processing_unavailable",
"result": "transient"
}
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"detail": {
"code": "processing_unavailable",
"result": "transient"
}
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"detail": {
"code": "processing_unavailable",
"result": "transient"
}
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"detail": {
"code": "processing_unavailable",
"result": "transient"
}
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"detail": {
"result": "ready",
"seconds": 0.2960601250001673,
"cpu_seconds": 0.26317221,
"rss_peak_mib": 79.25
}
},
{
"job_id": "20c34eb3-0582-4144-847c-fa037647f863",
"detail": {
"result": "ready",
"seconds": 0.3045327089998864,
"cpu_seconds": 0.2630334579999998,
"rss_peak_mib": 79.578125
}
},
{
"job_id": "c013ea06-4dea-4e65-b715-8a3f5976fa57",
"detail": {
"result": "ready",
"seconds": 0.3056550410001364,
"cpu_seconds": 0.26324616899999986,
"rss_peak_mib": 79.65234375
}
},
{
"job_id": "44de0dbe-f2b8-40bd-8b65-479e8f7243b5",
"detail": {
"code": "worker_timeout",
"result": "transient"
}
},
{
"job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9",
"detail": {
"result": "ready",
"seconds": 0.26621149999982663,
"cpu_seconds": 0.23299866699999994,
"rss_peak_mib": 79.67578125
}
},
{
"job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9",
"detail": {
"result": "ready",
"seconds": 0.29709320799975103,
"cpu_seconds": 0.24198554100000003,
"rss_peak_mib": 79.61328125
}
},
{
"job_id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46",
"detail": {
"result": "stale"
}
},
{
"job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8",
"detail": {
"result": "ready",
"seconds": 0.28706112500003655,
"cpu_seconds": 0.24108328799999978,
"rss_peak_mib": 79.671875
}
},
{
"job_id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf",
"detail": {
"result": "stale"
}
},
{
"job_id": "8eedc435-5509-4724-9a18-e478d920f101",
"detail": {
"result": "ready",
"seconds": 0.32150650000039604,
"cpu_seconds": 0.28338829200000015,
"rss_peak_mib": 79.37109375
}
}
],
"audit": [
{
"job_id": "cc18c75e-8886-4333-bae8-5da93d626c73",
"action": "new_version",
"operator": "synthetic-verifier",
"reason": "Explicit fault matrix replay",
"created_at": "2026-11-15 19:46:27.434253+00:00"
},
{
"job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1",
"action": "replay",
"operator": "synthetic-verifier",
"reason": "Explicit fault matrix replay",
"created_at": "2026-11-15 19:56:33.964020+00:00"
},
{
"job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9",
"action": "replay",
"operator": "synthetic-verifier",
"reason": "Explicit fault matrix replay",
"created_at": "2026-11-15 19:57:37.397099+00:00"
},
{
"job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9",
"action": "new_version",
"operator": "synthetic-verifier",
"reason": "Explicit fault matrix replay",
"created_at": "2026-11-15 19:59:07.104714+00:00"
},
{
"job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9",
"action": "new_version",
"operator": "synthetic-verifier",
"reason": "Held stale writer race",
"created_at": "2026-11-15 20:04:28.322925+00:00"
},
{
"job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8",
"action": "new_version",
"operator": "synthetic-verifier",
"reason": "Held stale writer race",
"created_at": "2026-11-15 20:05:28.102142+00:00"
}
]
}
+13 -4
View File
@@ -1,7 +1,16 @@
# Инфраструктура студента
# Инфраструктура ЛР5
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Топология и реальные pinned versions — compose.yaml; сборка MinIO/mc из
закреплённых commits — s3.Dockerfile; bucket/IAM — s3-bootstrap.sh и s3-policy.json.
Credentials создаёт scripts/bootstrap.py, значения не входят в Git/image.
API/worker/reconciler получают scoped application keys, relay только PG+broker,
root keys доступны только S3/bootstrap. Консоль S3 слушает loopback контейнера.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
Сервисы: postgres persistent volume; cache disposable; s3 persistent volume;
broker отдельный Valkey Streams persistent AOF volume; relay/outbox/watchdog;
worker отдельный2CPU1536MiB2slots; api2replicas; lb один localhost:8085.
S3/broker/PG/API host ports не публикуются. Сеть — private Compose default;
production TLS/изоляция от потери host не заявлены. Детали — REPORT.md/SCHEMA.md.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
compose.test.yaml включает fault instrumentation и отдельный controller,
без публичных endpoints; обычный запуск не имеет этого controller/hooks.
+107
View File
@@ -0,0 +1,107 @@
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
// go mod download validates its h1 checksum through the Go checksum database.
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func main() {
const version = "v0.0.0-20251015172955-9e49d5e7a648"
const total int64 = 25342606
const chunk int64 = 512 * 1024
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
if err := os.MkdirAll(directory, 0755); err != nil {
panic(err)
}
target := filepath.Join(directory, version+".zip")
if info, err := os.Stat(target); err == nil && info.Size() == total {
fmt.Println("Source ZIP cached; Go will validate checksum")
return
}
client := &http.Client{Timeout: 90 * time.Second}
jobs := make(chan int64)
errors := make(chan error, 64)
var group sync.WaitGroup
for worker := 0; worker < 8; worker++ {
group.Add(1)
go func() {
defer group.Done()
for start := range jobs {
end := min(start+chunk, total) - 1
part := fmt.Sprintf("%s.part-%d", target, start)
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
continue
}
var last error
for attempt := 0; attempt < 4; attempt++ {
req, _ := http.NewRequest("GET", source, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
response, err := client.Do(req)
if err != nil {
last = fmt.Errorf("source range request failed")
continue
}
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
response.Body.Close()
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
continue
}
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
response.Body.Close()
if err != nil || int64(len(data)) != end-start+1 {
last = fmt.Errorf("incomplete source range")
continue
}
if err := os.WriteFile(part, data, 0644); err != nil {
last = err
continue
}
last = nil
fmt.Printf("Source part %d verified\n", start/chunk+1)
break
}
if last != nil {
errors <- last
}
}
}()
}
for start := int64(0); start < total; start += chunk {
jobs <- start
}
close(jobs)
group.Wait()
close(errors)
for err := range errors {
panic(err)
}
out, err := os.Create(target + ".partial")
if err != nil {
panic(err)
}
for start := int64(0); start < total; start += chunk {
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
if err != nil {
panic(err)
}
if _, err := io.Copy(out, part); err != nil {
panic(err)
}
part.Close()
}
if err := out.Close(); err != nil {
panic(err)
}
if err := os.Rename(target+".partial", target); err != nil {
panic(err)
}
}
+36
View File
@@ -0,0 +1,36 @@
global
log stdout format raw local0
maxconn 512
stats socket /tmp/admin.sock mode 600 level admin
defaults
mode http
log global
option httplog
timeout connect 1s
timeout client 15s
timeout server 8s
timeout http-request 10s
timeout http-keep-alive 5s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
server-template api 1-8 api:8080
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_key=$(cat /run/secrets/s3_access_key)
api_secret=$(cat /run/secrets/s3_secret_key)
# Credentials are passed only inside this isolated bootstrap container.
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
mc mb --ignore-existing local/messenger >/dev/null 2>&1
mc anonymous set none local/messenger >/dev/null 2>&1
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
printf '%s\n' 'Private bucket and scoped application policy ready'
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001
exec minio server /data --address :9000 --console-address 127.0.0.1:9001
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger/objects/*"]
}
]
}
+49
View File
@@ -0,0 +1,49 @@
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
FROM toolchain AS minio-build
WORKDIR /source
COPY infra/fetch-source.go /tmp/fetch-source.go
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go run /tmp/fetch-source.go && \
for attempt in 1 2 3 4; do \
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
test "$attempt" -lt 4 || exit 1; sleep 1; \
done && \
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
for attempt in 1 2 3; do \
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
test "$attempt" -lt 3 || exit 1; sleep 1; \
done
FROM toolchain AS mc-build
WORKDIR /source
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go build -trimpath -ldflags='-s -w' -o /out/mc .
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
RUN mkdir /data && chown 10001:10001 /data
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
EXPOSE 9000
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
COPY --from=mc-build /out/mc /usr/local/bin/mc
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
COPY infra/s3-policy.json /policy.json
USER 10001:10001
ENV MC_CONFIG_DIR=/tmp/mc
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE idempotency (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY(user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
+32
View File
@@ -0,0 +1,32 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD UNIQUE(chat_id,id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
error_code text,
uploader_instance text NOT NULL,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(chat_id,id),
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY(message_id,attachment_id),
UNIQUE(message_id,position),
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+54
View File
@@ -0,0 +1,54 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
ALTER TABLE attachments ADD COLUMN published_version integer;
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
);
CREATE TABLE jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
CREATE TABLE outbox (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz
);
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
size_bytes bigint NOT NULL, sha256 text NOT NULL,
width integer, height integer,
PRIMARY KEY(attachment_id,pipeline_version,name)
);
CREATE TABLE effect_publications (
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(attachment_id,pipeline_version)
);
CREATE TABLE pipeline_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
operator text NOT NULL, reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
CREATE TABLE pipeline_faults (
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
);
CREATE TABLE pipeline_events (
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+45
View File
@@ -0,0 +1,45 @@
[project]
name = "mtusi-messenger-lab5"
version = "1.0.0"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi==0.143.3",
"pydantic==2.14.1",
"uvicorn==0.54.0",
"psycopg[binary,pool]==3.3.6",
"argon2-cffi>=25,<26",
"redis==8.1.0",
"minio==7.2.20",
"pillow==13.0.0",
]
[dependency-groups]
dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validator==0.7.2", "psutil>=7,<8", "matplotlib>=3.10,<4"]
[tool.ruff]
line-length = 79
target-version = "py314"
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
[tool.ruff.lint.per-file-ignores]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
# Keep SQL statements together; the formatter controls executable statements.
"src/attachments.py" = ["E501"]
"src/pipeline.py" = ["E501"]
"src/worker.py" = ["E501"]
"src/operator.py" = ["E501"]
"src/reconcile.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
+128
View File
@@ -0,0 +1,128 @@
"""Summarize measured fault invariants and child resource samples."""
import argparse
import csv
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def effects(case):
for key in ["effect", "v1", "v2"]:
if key in case:
yield key, case[key]
for i, item in enumerate(case.get("effects", [])):
yield str(i + 1), item
def main():
p = argparse.ArgumentParser()
p.add_argument("input", type=Path)
p.add_argument("--output", required=True, type=Path)
a = p.parse_args()
d = json.loads(a.input.read_text())
assert d["complete"] and d.get("scope", "all") == "all"
rows = []
for case in d["cases"]:
for role, e in effects(case):
j = e["job"]
rows.append(
dict(
case=case["case"],
role=role,
job_id=j["id"],
version=j["pipeline_version"],
attempt=j["attempt"],
cycle=j["cycle"],
fence=j["fence"],
state=j["state"],
publications=e["publications"],
variant_rows=len(e["variants"]),
physical_keys=len(e["physical_keys"]),
)
)
a.output.mkdir(parents=True, exist_ok=True)
with (a.output / "effects.csv").open("w") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
ready = [r for r in rows if r["state"] == "ready"]
assert all(
r["publications"] == 1
and r["variant_rows"] == 4
and r["physical_keys"] == 4
for r in ready
)
stale = [r for r in rows if r["state"] == "obsolete"]
assert all(r["publications"] == 0 for r in stale)
job_ids = {r["job_id"] for r in rows}
samples = [
x["detail"]
for x in d["worker_metrics"]
if str(x["job_id"]) in job_ids
and x["detail"].get("result") == "ready"
and "rss_peak_mib" in x["detail"]
]
stats = {
"source_commit": d["source_commit"],
"cases": len(d["cases"]),
"observed_ready_effects": len(ready),
"observed_obsolete_effects": len(stale),
"all_ready_have_one_publication_and_four_variants": True,
"all_obsolete_have_zero_publications": True,
"successful_child_samples": len(samples),
"child_elapsed_range_seconds": [
min(x["seconds"] for x in samples),
max(x["seconds"] for x in samples),
],
"child_cpu_range_seconds": [
min(x["cpu_seconds"] for x in samples),
max(x["cpu_seconds"] for x in samples),
],
"child_peak_rss_range_mib": [
min(x["rss_peak_mib"] for x in samples),
max(x["rss_peak_mib"] for x in samples),
],
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
}
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
axes[0].bar(
range(1, len(rows) + 1),
[r["publications"] for r in rows],
color=[
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
],
)
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
axes[0].set(
xlabel="Effect observation (CSV row)",
ylabel="Committed publications",
title="One publication per ready version; stale v1 = 0",
yticks=[0, 1],
)
axes[1].scatter(
[x["seconds"] for x in samples],
[x["rss_peak_mib"] for x in samples],
color="#246b8e",
s=35,
alpha=0.8,
)
axes[1].set(
xlabel="Child elapsed, seconds",
ylabel="Child peak RSS, MiB",
title=f"{len(samples)} completed child samples (synthetic320×200)",
)
for ax in axes:
ax.grid(axis="y", alpha=0.2)
fig.savefig(a.output / "pipeline-results.png", dpi=150)
plt.close(fig)
print(json.dumps(stats, indent=2))
if __name__ == "__main__":
main()
+63
View File
@@ -0,0 +1,63 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(directory / "s3_root_user", secrets.token_hex(10))
create(directory / "s3_root_password", secrets.token_urlsafe(32))
create(directory / "s3_access_key", secrets.token_hex(10))
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
create(directory / "broker_password", secrets.token_urlsafe(32))
create(
directory / "broker_url",
"redis://:"
+ (directory / "broker_password").read_text().strip()
+ "@broker:6379/0",
)
create(
directory / "broker.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
+ (directory / "broker_password").read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
"""Явные операторские действия только над API текущего Compose-проекта."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def containers():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def alias(cid):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=2); "
"print(r.headers['X-Instance-Id'])"
)
return command("docker", "exec", cid, "python", "-c", code)
def wait_count(count):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
data = containers()
if len(data) == count and all(
c["State"].get("Health", {}).get("Status") == "healthy"
for c in data
):
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
time.sleep(3)
return
time.sleep(0.5)
raise TimeoutError("Replica readiness")
def main():
action = sys.argv[1]
data = containers()
if action == "stop":
target = os.environ["TARGET_INSTANCE_ID"]
matches = [c for c in data if alias(c["Id"]) == target]
if len(matches) != 1 or len(data) < 2:
raise RuntimeError("Target must identify one running replica")
command("docker", "stop", "--time", "10", matches[0]["Id"])
print("Selected API stopped")
elif action == "rolling":
count = len(data)
if count < 2:
raise RuntimeError("Need at least two replicas")
# Сначала добавляется готовая реплика, затем удаляется одна старая.
for old in data:
command(
"docker",
"compose",
"up",
"-d",
"--no-deps",
"--no-recreate",
"--scale",
f"api={count + 1}",
"api",
)
wait_count(count + 1)
command("docker", "stop", "--time", "10", old["Id"])
command("docker", "rm", old["Id"])
wait_count(count)
print("Sequential replacement completed")
else:
raise ValueError("Unknown action")
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
docker compose restart api postgres cache s3 broker relay worker lb
docker compose up -d --no-build --no-recreate --wait
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py rolling
+137
View File
@@ -0,0 +1,137 @@
import json
import os
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8085")
def command(args, *, input=None):
result = subprocess.run(
args, cwd=ROOT, input=input, capture_output=True, text=True
)
if result.returncode:
raise RuntimeError(
"Command failed: "
+ args[0]
+ "; "
+ result.stderr.replace(str(ROOT), "<repository>")
)
return result.stdout.strip()
def compose(*args):
return command(["docker", "compose", *args])
def sql(query, *, database="messenger"):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
database,
"-A",
"-t",
"-v",
"ON_ERROR_STOP=1",
"-c",
query,
)
def cache(command_name, *args):
script = (
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
'exec valkey-cli "$@"'
)
return compose(
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
)
def ready(base=BASE, *, timeout=30):
end = time.monotonic() + timeout
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
while time.monotonic() < end:
try:
if client.get("/health/ready").status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.2)
raise TimeoutError("API readiness")
def actor(client):
username = "verify_" + uuid4().hex[:20]
password = "Synthetic-" + uuid4().hex + ":🌍"
result = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert result.status_code == 201
user = result.json()
session = client.post(
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
)
assert session.status_code == 201
return user, password, session.json()
def auth(session):
return {"Authorization": "Bearer " + session["token"]}
def extra_api(*, ttl=3600, pool_size=1):
name = "mtusi-lab5-test-" + uuid4().hex[:10]
compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"-e",
"SESSION_TTL_SECONDS=" + str(ttl),
"-e",
"DB_POOL_SIZE=" + str(pool_size),
"-p",
"127.0.0.1::8080",
"api",
)
try:
info = json.loads(command(["docker", "inspect", name]))[0]
base = (
"http://127.0.0.1:"
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
)
ready(base)
return name, base
except Exception:
command(["docker", "rm", "-f", name])
raise
def instances():
data = {}
for role in ["api", "postgres", "cache"]:
cid = compose("ps", "-q", role)
info = json.loads(command(["docker", "inspect", cid]))[0]
data[role] = {
"container": cid[:12],
"pid": info["State"]["Pid"],
"started_at": info["State"]["StartedAt"],
}
return data
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py stop
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
docker compose stop --timeout 1 worker
+254
View File
@@ -0,0 +1,254 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def run(args, *, cwd, env, expected=0):
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
def redact(text):
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
raise RuntimeError(redact(result.stdout + result.stderr))
return redact(result.stdout.strip())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab5-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab5-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab5-clean-") as temporary:
clone = Path(temporary) / "lab5"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
for command_args in [
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"seed",
"--seed",
"evidence/clean-gc-seed.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_pipeline",
"--output",
"evidence/clean-pipeline.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"verify",
"--seed",
"evidence/clean-gc-seed.json",
"--output",
"evidence/clean-cleanup.json",
],
]:
run(command_args, cwd=clone, env=env)
receipt["pipeline"] = json.loads(
(clone / "evidence/clean-pipeline.json").read_text()
)
receipt["cleanup"] = json.loads(
(clone / "evidence/clean-cleanup.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
ids += run(
[
"docker",
"compose",
"ps",
"-q",
"worker",
"relay",
"reconciler",
"controller",
],
cwd=clone,
env=env,
).split()
assert len(ids) == 6
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt[
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
finally:
assert project.startswith("mtusi-lab5-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()
+223
View File
@@ -0,0 +1,223 @@
"""Real S3 retention test: seed is separate from verification output."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
from scripts.verify_pipeline import inside, operator, wait
def seed(path):
if path.exists():
raise ValueError("Choose a new seed file")
ready()
compose("stop", "--timeout", "1", "reconciler")
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
user, _, session = actor(c)
headers = auth(session)
result = c.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Cleanup synthetic seed",
"member_ids": [user["id"]],
},
)
assert result.status_code == 201
chat = result.json()["id"]
r = c.post(
f"/api/v1/chats/{chat}/attachments",
headers={**headers, "Content-Type": "image/png"},
params={"filename": "retained.png"},
content=raw,
)
assert r.status_code == 202
attachment = r.json()["id"]
wait(
lambda: (
c.get(
"/api/v1/attachments/" + attachment, headers=headers
).json()["status"]
== "ready"
)
)
job = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
data = {
"seed_id": uuid4().hex,
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"ready_attachment": attachment,
"ready_job": job,
"chat": chat,
"owner": user["id"],
}
# Real old v1 objects will become obsolete after v2 is ready.
second = operator(job, version=2)
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
)
)
data["version2_job"] = second
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
data["failed_id"] = str(uuid4())
data["active_id"] = str(uuid4())
data["reservation_id"] = str(uuid4())
data["expired_reservation_id"] = str(uuid4())
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
data["active_source"] = f"objects/staging/{data['active_id']}/original"
data["reservation_key"] = (
f"objects/staging/{data['reservation_id']}/original"
)
data["expired_reservation_key"] = (
f"objects/staging/{data['expired_reservation_id']}/original"
)
inside(f"""from pathlib import Path
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
with db.connection() as c, c.transaction():
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
key='objects/staging/'+identity+'/original'
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
print('true')""")
data["seeded_unix"] = time.time()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: retention seed created; real S3 age must reach300s", flush=True
)
def verify(seed_path, output):
if output.exists():
raise ValueError("Choose a new output path")
data = json.loads(seed_path.read_text())
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"seed": seed_path.name,
"complete": False,
}
try:
compose("stop", "--timeout", "1", "reconciler")
def age():
return inside(
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
)
while age() < 302:
time.sleep(min(5, 303 - age()))
report["real_s3_age_seconds"] = age()
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
inside(
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
)
oldkeys = [
data["orphan"],
data["failed_source"],
data["expired_reservation_key"],
] + [
f"objects/{data['ready_attachment']}/v1/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
protected = [data["active_source"], data["reservation_key"], fresh] + [
f"objects/{data['ready_attachment']}/v2/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
before = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(before.values())
dry = inside(
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
)
candidates = {a["key"] for a in dry["actions"]}
assert set(oldkeys) <= candidates and not set(protected) & candidates
afterdry = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert afterdry == before
applied = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
repeat = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
assert not repeat["actions"]
after = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(not after[k] for k in oldkeys) and all(
after[k] for k in protected
)
failed = inside(
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
)
assert failed["error_code"] == "source_expired"
try:
operator(failed["id"])
except RuntimeError:
replay_refused = True
else:
raise AssertionError("Expired source replay must fail")
# Expired reservation was removed, so a delayed API cannot commit its handoff.
fenced = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
)
assert fenced
report.update(
dry_run=dry,
apply=applied,
repeat=repeat,
before=before,
after=after,
dry_run_unchanged=True,
active_current_and_fresh_protected=True,
expired_source_replay_refused=replay_refused,
expired_writer_fenced=fenced,
complete=True,
)
finally:
# Remove only this fixture's artificial active reservation/job; never real jobs.
inside(
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
)
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
flush=True,
)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["seed", "verify"])
p.add_argument("--seed", required=True, type=Path)
p.add_argument("--output", type=Path)
a = p.parse_args()
if a.action == "seed":
seed(a.seed)
else:
if a.output is None:
p.error("output required")
verify(a.seed, a.output)
if __name__ == "__main__":
main()
+347
View File
@@ -0,0 +1,347 @@
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while (
time.monotonic() - started < seconds
or not finished.is_set()
or time.monotonic() - started
< action_result.get("finished", 0) + 2
):
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "s3", "broker", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()
+687
View File
@@ -0,0 +1,687 @@
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
import argparse
import hashlib
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
PREFIX = """import io,json,hashlib,time,urllib.request
from uuid import UUID,uuid4
from psycopg.types.json import Jsonb
from src.pipeline import database,broker,envelope,STREAM,GROUP
from src.object_store import ObjectStore,BUCKET
from src.operator import change
from src.reconcile import reconcile
db,store,queue=database(),ObjectStore(),broker()
"""
def inside(code):
cid = compose("ps", "-q", "controller").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
)
)
def wait(fn, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = fn()
if value:
return value
time.sleep(0.15)
raise TimeoutError("Fault scenario condition not reached")
def stop(role):
compose("stop", "--timeout", "1", role)
def start(role):
compose("up", "-d", "--no-deps", "--no-recreate", role)
def row(job):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
)
def inject(job, phase, action, remaining=1):
return inside(
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
)
def hit(identity):
return inside(
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
)
def operator(job, version=None):
return inside(
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
)
def duplicate(job, count=5):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
)
def inspect_effect(job, *, pending=False):
return inside(f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
pending_entries=[]
if {pending!r}:
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
entry=item['message_id']
messages=queue.xrange(STREAM,min=entry,max=entry)
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument(
"--start-at", choices=["all", "versions"], default="all"
)
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise ValueError("Choose a new output path")
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"track": 2,
"scope": args.start_at,
"complete": False,
"cases": [],
}
output.parent.mkdir(parents=True, exist_ok=True)
def save(name, details):
report["cases"].append({"case": name, **details})
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS:", name, flush=True)
try:
ready()
# Check instrumentation is really enabled and production default is off.
cid = compose("ps", "-q", "api").split()[0]
assert (
command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import os;print(os.getenv('PIPELINE_FAULTS'))",
]
)
== "1"
)
stop("reconciler")
with httpx.Client(
base_url=BASE, trust_env=False, timeout=25
) as client:
user, _, session = actor(client)
h = auth(session)
c = client.post(
"/api/v1/chats",
headers=h,
json={
"title": "Pipeline synthetic faults",
"member_ids": [user["id"]],
},
)
assert c.status_code == 201
chat = c.json()["id"]
raw_io = io.BytesIO()
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
raw_io, format="PNG"
)
raw = raw_io.getvalue()
def upload():
r = client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "synthetic.png"},
content=raw,
)
assert r.status_code == 202, r.text
a = r.json()
assert a["status"] == "queued" and not a["variants"]
j = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
return a, j
def metadata(a):
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
assert r.status_code == 200, r.text
return r.json()
def until(a, state="ready", timeout=90):
return wait(
lambda: (
r if (r := metadata(a))["status"] == state else None
),
timeout,
)
def confirm(a, j):
until(a)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.status_code == 200 and downloaded.content == raw
)
m = metadata(a)
assert (
m["sha256"] == hashlib.sha256(raw).hexdigest()
and m["error_code"] is None
)
assert {v["name"] for v in m["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
for v in m["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=h,
params={"variant": v["name"]},
)
assert (
r.status_code == 200
and hashlib.sha256(r.content).hexdigest()
== v["sha256"]
)
with Image.open(io.BytesIO(r.content)) as image:
assert (
image.size
== {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (320, 200),
}[v["name"]]
)
assert not image.getexif()
effect = inspect_effect(j)
assert (
effect["publications"] == 1
and len(effect["variants"]) == 4
and len(effect["physical_keys"]) == 4
)
return effect
if args.start_at == "all":
# Source-only crash before the DB handoff must never be reported as accepted.
stop("worker")
stop("relay")
marker = inject(chat, "api_after_staging", "wait")
before = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(
lambda: client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "unconfirmed.png"},
content=raw,
)
)
wait(lambda marker=marker: hit(marker))
compose("kill", "-s", "SIGKILL", "api")
try:
response = future.result(15)
assert response.status_code != 202
except httpx.HTTPError:
pass
pool.shutdown()
start("api")
ready()
after = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
assert before == after
save(
"staging_before_handoff_crash",
{
"no_202": True,
"no_attachment_or_job_created": True,
"orphan_retained_for_cleanup": True,
},
)
# API durable commit/outbox survives restart without a broker or worker.
stop("broker")
a, j = upload()
linked = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert linked.status_code == 201
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
compose("restart", "api")
ready()
start("broker")
start("relay")
start("worker")
effect = confirm(a, j)
save(
"commit_before_publish_and_worker_stopped",
{
"attachment": a["id"],
"effect": effect,
"queued_link_allowed": True,
"api_and_broker_restarted": True,
"broker_stopped_during_upload": True,
},
)
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
stop("worker")
a, j = upload()
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
)
)
stop("relay")
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
wait(lambda: inside("print(json.dumps(queue.ping()))"))
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
assert restored == ids, {"before": ids, "after": restored}
start("relay")
start("worker")
save(
"confirmed_broker_entry_survives_sigkill",
{
"effect": confirm(a, j),
"same_stream_entries": True,
"entry_count": len(ids),
},
)
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
stop("worker")
stop("relay")
a, j = upload()
marker = inject(j, "relay_after_publish", "crash")
start("relay")
wait(lambda marker=marker: hit(marker))
time.sleep(0.5)
unpublished = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
)
assert unpublished >= 1
start("relay")
start("worker")
save(
"relay_crash_after_xadd_before_pg_marker",
{
"unpublished_after_crash": unpublished,
"effect": confirm(a, j),
},
)
# Repeated before/after effect and commit/ack windows.
for repeat in range(args.repeats):
for phase in [
"before_effect",
"after_effect",
"after_commit",
]:
stop("worker")
a, j = upload()
marker = inject(j, phase, "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
at_crash = inspect_effect(
j, pending=phase == "after_commit"
)
assert at_crash["job"]["state"] == (
"ready"
if phase == "after_commit"
else "processing"
)
assert at_crash["publications"] == (
1 if phase == "after_commit" else 0
)
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
if phase == "after_commit":
assert at_crash["pending_stream_entries"], (
"Result must be committed while notification is still unacknowledged"
)
assert not any(
e["event"] == "child_finished"
for e in at_crash["events"]
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
if phase == "after_commit":
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
and e["detail"]["state"] == "ready"
for e in inspect_effect(j)["events"]
)
)
wait(
lambda j=j: inside(
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
)
)
effect = inspect_effect(j)
else:
assert effect["job"]["attempt"] >= 2
save(
f"{phase}_sigkill_repeat_{repeat + 1}",
{"at_crash": at_crash, "effect": effect},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient")
start("worker")
effect = confirm(a, j)
assert effect["job"]["attempt"] == 2
save("transient_retry_with_jitter", {"effect": effect})
stop("worker")
a, j = upload()
marker = inject(j, "before_effect", "hang")
start("worker")
wait(lambda: hit(marker))
limits = inside(
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
)
assert limits["address_space_bytes"] == [805306368, 805306368]
assert limits["memory.max"] == "1610612736"
effect = confirm(a, j)
assert any(
e["event"] == "retry"
and e["detail"]["code"] == "worker_timeout"
for e in effect["events"]
)
save(
"hung_child_killed_and_retried",
{"effect": effect, "observed_limits": limits},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "permanent")
start("worker")
failed = until(a, "failed")
assert failed["error_code"] == "invalid_image"
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert r.status_code == 409
assert operator(j) == j
effect = confirm(a, j)
assert effect["job"]["cycle"] == 2
try:
operator(j)
except RuntimeError:
pass
else:
raise AssertionError("Replay of ready must be rejected")
save(
"permanent_failed_registry_and_audited_replay",
{
"failed": failed,
"effect": effect,
"ready_replay_rejected": True,
},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient", remaining=5)
start("worker")
failed = until(a, "failed", timeout=150)
assert (
failed["error_code"] == "retry_exhausted"
and row(j)["attempt"] == 5
)
operator(j)
save(
"finite_retry_exhaustion_then_explicit_replay",
{"failed": failed, "effect": confirm(a, j)},
)
# Reverse notification order and add concurrent copies: one publication/key.
stop("worker")
stop("relay")
a, j = upload()
b, k = upload()
duplicate(k, 8)
duplicate(j, 8)
start("worker")
start("relay")
ea = confirm(a, j)
confirm(b, k)
duplicate(j, 10)
duplicate(k, 10)
wait(
lambda: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert (
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
)
save(
"reordered_and_duplicate_notifications",
{
"effects": [inspect_effect(j), inspect_effect(k)],
"added_duplicates": 36,
"no_extra_attempt_after_ready": True,
},
)
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
for repeat in range(args.repeats):
stop("worker")
a, j = upload()
marker = inject(j, "after_effect", "wait")
# Keep one controller process hot throughout the narrow stale-writer window.
code = f"""from src.pipeline import event
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
end=time.monotonic()+45
while True:
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
if hit: break
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
time.sleep(.05)
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
end=time.monotonic()+10
while True:
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
if ready: break
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
time.sleep(.05)
with db.connection() as c:
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
end=time.monotonic()+10
while True:
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
if stale: break
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
time.sleep(.05)
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
"""
with ThreadPoolExecutor(max_workers=1) as hot:
future = hot.submit(inside, code)
wait(
lambda j=j: inside(
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
)
)
start("worker")
race = future.result(timeout=45)
k = race["job_id"]
hashes = race["hashes"]
confirm(a, k)
duplicate(j, 12)
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert hashes == {
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
}
assert inspect_effect(j)["publications"] == 0
save(
f"stale_v1_after_v2_repeat_{repeat + 1}",
{
"v1": inspect_effect(j),
"v2": inspect_effect(k),
"new_hashes_unchanged": True,
},
)
# No arbitrary URL/bytes fields in queue envelopes.
invalid = inside(
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
)
assert invalid
wait(
lambda: inside(
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
)
)
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
# Private bucket, bounded IAM, and download after uploader process dies.
a, j = upload()
confirm(a, j)
key = f"objects/{a['id']}/v1/original"
security = inside(f"""from minio.error import S3Error
from urllib.error import HTTPError
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
except HTTPError as e: anonymous=e.code
else: raise AssertionError('Anonymous access allowed')
denials={{}}
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
try: fn()
except S3Error as e: denials[name]=e.code
else: raise AssertionError('IAM allowed forbidden action')
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
assert security["anonymous"] == 403 and set(
security["iam"].values()
) == {"AccessDenied"}
selected = None
for cid in compose("ps", "-q", "api").split():
alias = command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
]
)
uploader = inside(
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
)
if alias == uploader:
selected = cid
assert selected
command(["docker", "stop", "--time", "1", selected])
ready()
wait(
lambda: (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 200
)
)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.content == raw
and downloaded.headers["X-Instance-Id"] != uploader
)
start("api")
ready()
save(
"private_iam_and_other_api_download",
{
"security": security,
"attachment": a["id"],
"uploader": uploader,
"downloader": downloaded.headers["X-Instance-Id"],
"sha256": hashlib.sha256(raw).hexdigest(),
},
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)
report["audit"] = inside(
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
)
report["complete"] = True
finally:
for role in ["api", "broker", "relay", "worker", "reconciler"]:
try:
start(role)
except Exception:
pass
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
if __name__ == "__main__":
main()
View File
+44
View File
@@ -0,0 +1,44 @@
import json
import logging
import os
import signal
import sys
def early_stop(signum, _frame):
print(
json.dumps({"event": "stopped_before_startup", "signal": signum}),
flush=True,
)
sys.exit(0)
signal.signal(signal.SIGTERM, early_stop)
signal.signal(signal.SIGINT, early_stop)
import uvicorn # noqa: E402
from src.logging_config import JsonFormatter # noqa: E402
def main() -> None:
host = os.environ.get("APP_HOST", "127.0.0.1")
port = int(os.environ.get("APP_PORT", "8080"))
if not 1 <= port <= 65535:
raise ValueError("APP_PORT must be between 1 and 65535")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
uvicorn.run(
"src.api:app",
host=host,
port=port,
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
log_config=None,
)
if __name__ == "__main__":
main()
+424
View File
@@ -0,0 +1,424 @@
import hashlib
import json
import logging
import tempfile
import time
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import datetime
from functools import partial
from urllib.parse import quote
from uuid import UUID, uuid4
from anyio import Semaphore, fail_after, to_thread
from fastapi import FastAPI, Request
from fastapi.encoders import jsonable_encoder
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from starlette.requests import ClientDisconnect
from starlette.responses import JSONResponse, Response, StreamingResponse
from src import images
from src.attachments import Attachments
from src.cache import UserCache
from src.cursor import parse_uuid
from src.database import Database
from src.domain import timestamp
from src.errors import DomainError, invalid
from src.object_store import ObjectStore
from src.password import Passwords
from src.repository import PostgresRepository
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
from src.service import MessengerService
from src.settings import Settings
logger = logging.getLogger("messenger")
INSTANCE_ID = uuid4().hex
def response(data: object, *, status: int = 200) -> JSONResponse:
return JSONResponse(
jsonable_encoder(
data, custom_encoder={datetime: timestamp, UUID: str}
),
status_code=status,
)
async def body(request: Request, model: type[Input]) -> Input:
try:
data = await request.body()
return model.model_validate(
json.loads(
data.decode("utf-8"),
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
)
)
except ValueError, ValidationError, UnicodeError, RecursionError:
raise invalid() from None
def pagination(request: Request) -> tuple[int, str | None]:
limits = request.query_params.getlist("limit")
cursors = request.query_params.getlist("cursor")
if len(limits) > 1 or len(cursors) > 1:
raise invalid()
raw = limits[0] if limits else "50"
if not raw.isascii() or not raw.isdecimal():
raise invalid()
raw = raw.lstrip("0") or "0"
if len(raw) > 3:
raise invalid()
limit = int(raw)
if not 1 <= limit <= 100:
raise invalid()
return limit, cursors[0] if cursors else None
def service(request: Request) -> MessengerService:
return request.app.state.service
async def run_io(function, **kwargs):
return await to_thread.run_sync(partial(function, **kwargs))
async def actor(request: Request):
return await run_io(
service(request).identify, value=request.headers.get("authorization")
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
settings = Settings.load()
database = Database(url=settings.database_url, size=settings.pool_size)
cache = UserCache(url=settings.cache_url)
store = ObjectStore()
try:
await run_io(database.open)
passwords = await run_io(Passwords)
app.state.database = database
app.state.store = store
app.state.upload_slots = Semaphore(2)
app.state.attachments = Attachments(
database=database, store=store, instance=INSTANCE_ID
)
app.state.service = MessengerService(
repository=PostgresRepository(database=database, cache=cache),
passwords=passwords,
session_ttl=settings.session_ttl,
)
app.state.ready = True
yield
finally:
app.state.ready = False
await run_io(store.close)
await run_io(cache.close)
await run_io(database.close)
app = FastAPI(
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
)
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid4())
request.state.request_id = request_id
started = time.perf_counter()
try:
result = await call_next(request)
except Exception:
logger.error(
json.dumps({"event": "internal_error", "request_id": request_id})
)
result = response(
{
"error": {
"code": "internal_error",
"message": "Internal error",
"request_id": request_id,
}
},
status=500,
)
result.headers["X-Request-Id"] = request_id
result.headers["X-Instance-Id"] = INSTANCE_ID
route = request.scope.get("route")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request_id,
"method": request.method,
"route": getattr(route, "path", "unmatched"),
"status": result.status_code,
"elapsed_ms": round((time.perf_counter() - started) * 1000, 3),
}
)
)
return result
@app.exception_handler(DomainError)
async def domain_error(request: Request, error: DomainError):
result = response(
{
"error": {
"code": error.code,
"message": error.message,
"request_id": request.state.request_id,
}
},
status=error.status,
)
result.headers.update(error.headers)
return result
@app.exception_handler(HTTPException)
async def http_error(request: Request, error: HTTPException):
code = "not_found" if error.status_code == 404 else "invalid_request"
return response(
{
"error": {
"code": code,
"message": "Resource not found"
if error.status_code == 404
else "Invalid request",
"request_id": request.state.request_id,
}
},
status=error.status_code,
)
@app.get("/health/live")
async def live():
return response({"status": "ok"})
@app.get("/health/ready")
async def ready():
if not getattr(app.state, "ready", False):
raise DomainError(503, "unavailable", "Not ready")
await run_io(app.state.database.ping)
await run_io(app.state.store.ping)
return response({"status": "ok"})
@app.post("/api/v1/users")
async def register(request: Request):
data = await body(request, CreateUser)
return response(
await run_io(service(request).register, **data.model_dump()),
status=201,
)
@app.get("/api/v1/users/me")
async def current(request: Request):
return response(await actor(request))
@app.post("/api/v1/chats")
async def create_chat(request: Request):
user = await actor(request)
data = await body(request, CreateChat)
return response(
await run_io(
service(request).create_chat, actor_id=user.id, **data.model_dump()
),
status=201,
)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).chats,
actor_id=user.id,
limit=limit,
cursor=cursor,
)
)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(request: Request, chat_id: str):
user = await actor(request)
return response(
await run_io(
service(request).get_chat,
actor_id=user.id,
chat_id=parse_uuid(chat_id),
)
)
@app.post("/api/v1/chats/{chat_id}/messages")
async def send(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
data = await body(request, CreateMessage)
return response(
await run_io(
service(request).send,
actor_id=user.id,
chat_id=cid,
idempotency_key=request.headers.get("idempotency-key"),
**data.model_dump(),
),
status=201,
)
@app.get("/api/v1/chats/{chat_id}/messages")
async def history(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
chat = await run_io(
service(request).get_chat, actor_id=user.id, chat_id=cid
)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).history,
actor_id=user.id,
chat=chat,
limit=limit,
cursor=cursor,
)
)
@app.post("/api/v1/auth/sessions")
async def login(request: Request):
return response(
await run_io(
service(request).login,
authorization=request.headers.get("authorization"),
),
status=201,
)
@app.delete("/api/v1/auth/sessions/current")
async def logout(request: Request):
await run_io(
service(request).logout,
authorization=request.headers.get("authorization"),
)
return Response(status_code=204)
@app.post("/api/v1/chats/{chat_id}/attachments")
async def upload(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
names = request.query_params.getlist("filename")
if len(names) != 1:
raise invalid()
name = images.filename(names[0])
declared = (
request.headers.get("content-type", "")
.partition(";")[0]
.strip()
.lower()
)
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
length = request.headers.get("content-length")
if length and (not length.isdecimal() or len(length) > 12):
raise invalid()
if length and int(length) > images.MAX_BYTES:
raise DomainError(413, "payload_too_large", "Byte limit exceeded")
try:
with fail_after(20):
async with app.state.upload_slots:
with tempfile.SpooledTemporaryFile(
max_size=1024 * 1024, mode="w+b"
) as stream:
size = 0
digest = hashlib.sha256()
async for chunk in request.stream():
size += len(chunk)
if size > images.MAX_BYTES:
raise DomainError(
413, "payload_too_large", "Byte limit exceeded"
)
digest.update(chunk)
stream.write(chunk)
await run_io(
images.verify, stream=stream, declared=declared
)
result = await run_io(
app.state.attachments.upload,
chat_id=cid,
owner_id=user.id,
filename=name,
content_type=declared,
size=size,
sha256=digest.hexdigest(),
stream=stream,
)
return response(result, status=202)
except ClientDisconnect:
raise invalid() from None
except TimeoutError:
raise DomainError(
503, "unavailable", "Upload deadline exceeded"
) from None
@app.get("/api/v1/attachments/{attachment_id}")
async def attachment_metadata(request: Request, attachment_id: str):
user = await actor(request)
return response(
await run_io(
app.state.attachments.metadata,
identity=parse_uuid(attachment_id),
actor_id=user.id,
)
)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def attachment_content(request: Request, attachment_id: str):
user = await actor(request)
variants = request.query_params.getlist("variant")
if len(variants) > 1:
raise invalid()
row, upstream = await run_io(
app.state.attachments.content,
identity=parse_uuid(attachment_id),
actor_id=user.id,
variant=variants[0] if variants else "original",
)
def chunks():
try:
while data := upstream.read(65536):
yield data
finally:
upstream.close()
upstream.release_conn()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "inline; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
},
)
+144
View File
@@ -0,0 +1,144 @@
from uuid import uuid4
from src.errors import DomainError, not_found
from src.pipeline import enqueue, fault
def dto(row, variants=()):
return {
k: row[k]
for k in [
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"created_at",
"updated_at",
]
} | {
"status": row["state"],
"variants": [
{
k: v[k]
for k in [
"name",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
]
}
for v in variants
if v["name"] != "original"
],
"error_code": row["error_code"] if row["state"] == "failed" else None,
}
class Attachments:
def __init__(self, *, database, store, instance):
self.database, self.store, self.instance = database, store, instance
def upload(
self,
*,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
stream,
):
identity, job_id = uuid4(), uuid4()
key = f"objects/staging/{identity}/original"
with self.database.connection() as conn:
conn.execute(
"INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)",
(identity, key),
)
self.store.put(
key=key,
stream=stream,
size=size,
content_type=content_type,
sha256=sha256,
)
fault(self.database, chat_id, "api_after_staging")
with self.database.connection() as conn, conn.transaction():
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE",
(identity,),
).fetchone()
if reservation is None:
raise DomainError(
503, "unavailable", "Upload reservation expired"
)
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) "
"VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *",
(
identity,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
key,
self.instance,
),
).fetchone()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')",
(job_id, identity, key),
)
enqueue(conn, job_id)
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s", (identity,)
)
fault(self.database, chat_id, "api_after_commit")
# Captured committed snapshot remains queued even if worker already finished.
return dto(row)
def row(self, *, identity, actor_id):
with self.database.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s",
(actor_id, identity),
).fetchone()
if row is None:
raise not_found()
return row
def metadata(self, *, identity, actor_id):
row = self.row(identity=identity, actor_id=actor_id)
variants = []
if row["state"] == "ready":
with self.database.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
(identity, row["published_version"]),
).fetchall()
return dto(row, variants)
def content(self, *, identity, actor_id, variant):
row = self.row(identity=identity, actor_id=actor_id)
if row["state"] != "ready":
raise DomainError(409, "conflict", "Attachment is not ready")
with self.database.connection() as conn:
data = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(identity, row["published_version"], variant),
).fetchone()
if data is None:
raise not_found()
if self.store.head(key=data["object_key"]) is None:
raise DomainError(503, "unavailable", "Variant unavailable")
return {**data, "filename": row["filename"]}, self.store.get(
key=data["object_key"]
)
+66
View File
@@ -0,0 +1,66 @@
import json
import threading
import time
from datetime import datetime
from uuid import UUID
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from src.domain import User, timestamp
class UserCache:
def __init__(self, *, url: str):
self.client = redis.Redis.from_url(
url,
socket_connect_timeout=0.15,
socket_timeout=0.15,
max_connections=16,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
self.disabled_until = 0.0
self.lock = threading.Lock()
def close(self):
self.client.close()
def _command(self, function, *args, **kwargs):
if time.monotonic() < self.disabled_until:
return None
try:
return function(*args, **kwargs)
except redis.RedisError, OSError:
with self.lock:
self.disabled_until = time.monotonic() + 5
return None
def get(self, *, user_id: UUID) -> User | None:
raw = self._command(self.client.get, "user:" + str(user_id))
if raw is None:
return None
try:
row = json.loads(raw)
if UUID(row["id"]) != user_id:
return None
return User(
user_id,
row["username"],
row["display_name"],
datetime.fromisoformat(row["created_at"]),
)
except ValueError, TypeError, KeyError:
return None
def put(self, *, user: User):
raw = json.dumps(
{
"id": str(user.id),
"username": user.username,
"display_name": user.display_name,
"created_at": timestamp(user.created_at),
}
)
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
+64
View File
@@ -0,0 +1,64 @@
import base64
import binascii
import json
import re
from datetime import datetime
from uuid import UUID
from src.domain import Key, timestamp
from src.errors import invalid
def parse_uuid(value: str) -> UUID:
if not isinstance(value, str) or not re.fullmatch(
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
):
raise invalid()
return UUID(value)
def encode_cursor(*, scope: str, key: Key) -> str:
payload = {
"v": 1,
"scope": scope,
"at": timestamp(key[0]),
"id": str(key[1]),
}
return (
base64.urlsafe_b64encode(
json.dumps(payload, separators=(",", ":")).encode()
)
.decode()
.rstrip("=")
)
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
if value is None:
return None
try:
if (
not value
or len(value) > 2048
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
):
raise ValueError
data = json.loads(
base64.b64decode(
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
)
)
if (
not isinstance(data, dict)
or type(data.get("v")) is not int
or data["v"] != 1
or data.get("scope") != scope
or not isinstance(data.get("at"), str)
):
raise ValueError
at = datetime.fromisoformat(data["at"])
if at.tzinfo is None or timestamp(at) != data["at"]:
raise ValueError
return at, parse_uuid(data["id"])
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
raise invalid() from None
+69
View File
@@ -0,0 +1,69 @@
import time
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from src.errors import DomainError
class DeadlineConnection(psycopg.Connection):
"""Клиентский срок действует и при замороженном PostgreSQL."""
def wait(self, gen, interval=0.1, timeout=None):
deadline = getattr(self, "request_deadline", None)
remaining = (
max(0.001, deadline - time.monotonic()) if deadline else 3.0
)
bound = min(remaining, timeout) if timeout is not None else remaining
try:
return super().wait(gen, interval=interval, timeout=bound)
except psycopg.OperationalError:
self.close()
raise
class Database:
def __init__(self, *, url: str, size: int):
self.pool = ConnectionPool(
url,
connection_class=DeadlineConnection,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": (
"-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=6000"
),
},
min_size=1,
max_size=size,
timeout=1,
open=False,
)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
self.pool.close(timeout=2)
@contextmanager
def connection(self):
try:
with self.pool.connection(timeout=1) as conn:
conn.request_deadline = time.monotonic() + 4
try:
yield conn
finally:
conn.request_deadline = None
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
raise DomainError(
503, "unavailable", "Database unavailable"
) from None
def ping(self):
with self.connection() as conn:
conn.execute("SELECT 1").fetchone()
+45
View File
@@ -0,0 +1,45 @@
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
def utcnow() -> datetime:
return datetime.now(UTC)
def timestamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
type Record = User | Chat | Message
type Key = tuple[datetime, UUID]
def order_key(record: Record) -> Key:
return record.created_at, record.id
+16
View File
@@ -0,0 +1,16 @@
class DomainError(Exception):
def __init__(
self, status: int, code: str, message: str, headers=None
) -> None:
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid() -> DomainError:
return DomainError(400, "invalid_request", "Invalid request")
def not_found() -> DomainError:
return DomainError(404, "not_found", "Resource not found")
+77
View File
@@ -0,0 +1,77 @@
import struct
import threading
import unicodedata
import zlib
from PIL import Image, UnidentifiedImageError
from src.errors import DomainError, invalid
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
DECODE_SLOT = threading.BoundedSemaphore(1)
def filename(value):
if (
not value
or not 1 <= len(value) <= 255
or not value.strip()
or any(
c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"}
for c in value
)
):
raise invalid()
return value
def verify(stream, *, declared):
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
with DECODE_SLOT:
try:
stream.seek(0)
with Image.open(stream) as image:
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(
image.format
)
if actual != declared:
raise DomainError(
415, "unsupported_media_type", "Image type mismatch"
)
if image.width * image.height > MAX_PIXELS:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
)
image.verify()
# JPEG verify alone does not decode entropy data.
stream.seek(0)
with Image.open(stream) as image:
image.load()
except Image.DecompressionBombError:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
) from None
except (
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
struct.error,
zlib.error,
EOFError,
):
raise DomainError(
415, "unsupported_media_type", "Invalid image"
) from None
except MemoryError:
raise DomainError(
503, "unavailable", "Image memory budget exceeded"
) from None
finally:
stream.seek(0)
return declared
+17
View File
@@ -0,0 +1,17 @@
import json
import logging
from datetime import UTC, datetime
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
message = record.getMessage()
try:
event = json.loads(message)
if not isinstance(event, dict):
event = {"event": message}
except ValueError:
event = {"event": message}
event["timestamp"] = datetime.now(UTC).isoformat()
event["level"] = record.levelname.lower()
return json.dumps(event, ensure_ascii=False)
+51
View File
@@ -0,0 +1,51 @@
import hashlib
import json
from pathlib import Path
import psycopg
from src.settings import secret
def main():
with psycopg.connect(
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
) as conn:
conn.execute("SELECT pg_advisory_lock(2301002)")
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
for path in sorted(Path("migrations").glob("*.sql")):
content = path.read_bytes()
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s",
(path.name,),
).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError(
"Applied migration checksum mismatch"
)
continue
with conn.transaction():
conn.execute(content.decode())
conn.execute(
"INSERT INTO schema_migrations(name,checksum) "
"VALUES(%s,%s)",
(path.name, checksum),
)
print(
json.dumps(
{"event": "migration_applied", "name": path.name}
)
)
finally:
conn.execute("SELECT pg_advisory_unlock(2301002)")
if __name__ == "__main__":
main()
+84
View File
@@ -0,0 +1,84 @@
from contextlib import contextmanager
import urllib3
from minio import Minio
from minio.error import S3Error
from src.errors import DomainError
from src.settings import secret
BUCKET = "messenger"
class ObjectStore:
def __init__(self):
self.http = urllib3.PoolManager(
maxsize=32,
block=True,
timeout=urllib3.Timeout(connect=1, read=3),
retries=urllib3.Retry(total=0),
)
self.client = Minio(
"s3:9000",
access_key=secret("S3_ACCESS_KEY"),
secret_key=secret("S3_SECRET_KEY"),
secure=False,
region="us-east-1",
http_client=self.http,
)
@contextmanager
def operation(self):
try:
yield
except S3Error, urllib3.exceptions.HTTPError, OSError:
raise DomainError(
503, "unavailable", "Object store unavailable"
) from None
def ping(self):
with self.operation():
if not self.client.bucket_exists(BUCKET):
raise DomainError(
503, "unavailable", "Object store unavailable"
)
def put(self, *, key, stream, size, content_type, sha256):
stream.seek(0)
with self.operation():
self.client.put_object(
BUCKET,
key,
stream,
length=size,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10 * 1024 * 1024,
num_parallel_uploads=1,
)
def head(self, *, key):
with self.operation():
try:
return self.client.stat_object(BUCKET, key)
except S3Error as exc:
if exc.code in {"NoSuchKey", "NoSuchObject"}:
return None
raise
def get(self, *, key):
with self.operation():
return self.client.get_object(BUCKET, key)
def delete(self, *, key):
with self.operation():
self.client.remove_object(BUCKET, key)
def objects(self):
with self.operation():
yield from self.client.list_objects(
BUCKET, prefix="objects/", recursive=True
)
def close(self):
self.http.clear()
+123
View File
@@ -0,0 +1,123 @@
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
import argparse
import json
from uuid import UUID, uuid4
from src.object_store import ObjectStore
from src.pipeline import database, enqueue, event
def change(db, store, *, job_id, operator, reason, version=None):
if (
not operator.strip()
or not reason.strip()
or len(operator) > 100
or len(reason) > 500
):
raise ValueError("Operator and reason are required")
with db.connection() as conn, conn.transaction():
job = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
if not job:
raise ValueError("Unknown job")
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
(job["attachment_id"],),
).fetchone()
if job["pipeline_version"] != attachment["pipeline_version"]:
raise ValueError("Superseded job cannot be replayed")
source = (
job["source_key"]
if attachment["published_version"] is None
else attachment["object_key"]
)
if store.head(key=source) is None:
raise ValueError("Source expired or missing; replay refused")
if version is None:
if job["state"] != "failed":
raise ValueError("Only failed jobs can be replayed")
conn.execute(
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
action = "replay"
target = job_id
else:
if (
version not in {1, 2}
or version <= attachment["pipeline_version"]
):
raise ValueError("Supported version must increase (1 or 2)")
conn.execute(
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
(job_id,),
)
target = uuid4()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
(target, attachment["id"], version, source),
)
conn.execute(
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
(version, attachment["id"]),
)
action = "new_version"
conn.execute(
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment["id"],),
)
conn.execute(
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
(target, action, operator, reason),
)
event(conn, target, action, {"previous_job": str(job_id)})
enqueue(conn, target)
return str(target)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["failed", "replay", "version"])
p.add_argument("--job-id", type=UUID)
p.add_argument("--operator")
p.add_argument("--reason")
p.add_argument("--version", type=int)
a = p.parse_args()
db, store = database(), ObjectStore()
try:
if a.action == "failed":
with db.connection() as conn:
rows = conn.execute(
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
).fetchall()
print(json.dumps(rows, default=str))
else:
if not all([a.job_id, a.operator, a.reason]):
p.error("job-id/operator/reason required")
if a.action == "version" and a.version is None:
p.error("version required")
print(
json.dumps(
{
"job_id": change(
db,
store,
job_id=a.job_id,
operator=a.operator,
reason=a.reason,
version=a.version
if a.action == "version"
else None,
)
}
)
)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
import threading
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.low_level import Type
from src.errors import DomainError
class Passwords:
def __init__(self):
self.hasher = PasswordHasher(
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
)
self.slots = threading.BoundedSemaphore(2)
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
def _run(self, function, *args):
if not self.slots.acquire(timeout=5):
raise DomainError(503, "unavailable", "Authentication busy")
try:
return function(*args)
finally:
self.slots.release()
def hash(self, *, password: str) -> str:
return self._run(self.hasher.hash, password)
def verify(self, *, password: str, encoded: str | None) -> bool:
try:
return (
self._run(self.hasher.verify, encoded or self.dummy, password)
and encoded is not None
)
except VerificationError, InvalidHashError:
return False
+254
View File
@@ -0,0 +1,254 @@
"""Durable job state is authoritative; stream entries are wake-up notifications."""
import json
import os
import random
import re
import time
from datetime import UTC, datetime
from uuid import UUID
import redis
from psycopg.types.json import Jsonb
from src.database import Database
from src.settings import integer, secret
STREAM = "attachments"
GROUP = "image-workers"
def database():
db = Database(url=secret("DATABASE_URL"), size=4)
db.open()
return db
def broker():
return redis.Redis.from_url(
secret("BROKER_URL"),
decode_responses=True,
socket_connect_timeout=1,
socket_timeout=3,
max_connections=8,
)
def envelope(row):
return dict(
schema_version=1,
job_id=str(row["id"]),
type="attachment.process.v1",
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
)
def validate_job(data):
required = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
if (
not isinstance(data, dict)
or not required <= data.keys()
or data.keys() - required - {"traceparent", "tracestate"}
):
raise ValueError("Invalid job envelope")
if (
type(data["schema_version"]) is not int
or data["schema_version"] != 1
or data["type"] != "attachment.process.v1"
):
raise ValueError("Invalid job type")
for k in ["job_id", "attachment_id"]:
if not isinstance(data[k], str):
raise ValueError("Invalid job UUID")
UUID(data[k])
for k in ["pipeline_version", "attempt"]:
if type(data[k]) is not int or data[k] < 1:
raise ValueError("Invalid job number")
timestamp = data["created_at"]
if not isinstance(timestamp, str) or not re.fullmatch(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})",
timestamp,
):
raise ValueError("Invalid job timestamp")
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if "traceparent" in data:
if not isinstance(data["traceparent"], str) or not re.fullmatch(
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"]
):
raise ValueError("Invalid trace context")
if "tracestate" in data and (
not isinstance(data["tracestate"], str)
or len(data["tracestate"]) > 512
):
raise ValueError("Invalid trace context")
return data
def event(conn, job_id, name, detail=None):
conn.execute(
"INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)",
(job_id, name, Jsonb(detail or {})),
)
def enqueue(conn, job_id):
row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone()
conn.execute(
"INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)",
(job_id, Jsonb(envelope(row)), row["next_at"]),
)
def fault(db, target, phase):
if os.getenv("PIPELINE_FAULTS") != "1":
return
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1",
(target, phase),
).fetchone()
if row is None:
return
conn.execute(
"UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
action = row["action"]
if action == "crash":
os._exit(89)
if action == "permanent":
raise PermanentFailure("invalid_image")
if action == "transient":
raise RuntimeError("injected_transient")
if action == "hang":
time.sleep(120)
if action == "wait":
end = time.monotonic() + 60
while time.monotonic() < end:
with db.connection() as conn:
released = conn.execute(
"SELECT released_at FROM pipeline_faults WHERE id=%s",
(row["id"],),
).fetchone()
if released and released["released_at"]:
return
time.sleep(0.05)
raise RuntimeError("fault_wait_timeout")
class PermanentFailure(Exception):
pass
def retry(conn, row, error, *, permanent=False):
maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20)
failed = permanent or row["attempt"] >= maximum
delay = random.uniform(0.25, min(30, 2 ** row["attempt"]))
state = "failed" if failed else "queued"
code = error if permanent else ("retry_exhausted" if failed else error)
conn.execute(
"UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s",
(state, code, delay, row["id"]),
)
conn.execute(
"UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(
state,
code if failed else None,
row["attachment_id"],
row["pipeline_version"],
),
)
event(
conn,
row["id"],
"failed" if failed else "retry",
{"attempt": row["attempt"], "code": code, "backoff_seconds": delay},
)
if not failed:
enqueue(conn, row["id"])
def watchdog(db):
with db.connection() as conn, conn.transaction():
rows = conn.execute(
"SELECT * FROM jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
retry(conn, row, "lease_expired")
# Recreate a wakeup if its stream entry was lost: never recreate logical jobs.
rows = conn.execute(
"SELECT j.* FROM jobs j WHERE j.state='queued' AND j.next_at<=clock_timestamp() AND NOT EXISTS(SELECT 1 FROM outbox o WHERE o.job_id=j.id AND (o.published_at IS NULL OR o.published_at>clock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
enqueue(conn, row["id"])
def relay_once(db, queue):
watchdog(db)
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
).fetchone()
if not row:
return False
validate_job(row["payload"])
# WAITAOF must use the SAME connection as XADD (not two pooled calls).
with queue.client() as dedicated:
entry = dedicated.xadd(
STREAM,
{"job": json.dumps(row["payload"], separators=(",", ":"))},
)
confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000)
if int(confirmation[0]) < 1:
raise RuntimeError("Broker durability confirmation timed out")
fault(db, row["job_id"], "relay_after_publish")
conn.execute(
"UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
event(
conn,
row["job_id"],
"published",
{"entry": entry, "outbox": row["id"]},
)
return True
def relay():
db, queue = database(), broker()
try:
while True:
try:
if not relay_once(db, queue):
time.sleep(0.2)
except Exception:
print(
json.dumps(
{
"event": "relay_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(1)
finally:
queue.close()
db.close()
if __name__ == "__main__":
relay()
+122
View File
@@ -0,0 +1,122 @@
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
import argparse
import json
import time
from src.object_store import ObjectStore
from src.pipeline import database
from src.settings import integer
def reconcile(db, store, *, apply=False, grace=None):
grace = (
grace
if grace is not None
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
)
if grace < 300:
raise ValueError("Retention must be at least 300 seconds")
actions = []
for obj in store.objects():
# Recheck each candidate in a bounded transaction, including late PUT age.
with db.connection() as conn, conn.transaction():
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
"now"
]
if (now - obj.last_modified).total_seconds() < grace:
continue
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
(obj.object_name,),
).fetchone()
if reservation and reservation["expires_at"] > now:
continue
jobs = conn.execute(
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
(obj.object_name, obj.object_name),
).fetchall()
if any(j["state"] in {"queued", "processing"} for j in jobs):
continue
current = conn.execute(
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
(obj.object_name,),
).fetchone()
if current:
continue
if any(
j["state"] == "failed"
and (now - j["updated_at"]).total_seconds() < grace
for j in jobs
):
continue
head = store.head(key=obj.object_name)
if (
head is None
or (now - head.last_modified).total_seconds() < grace
):
continue
action = {
"action": "delete_expired_object",
"key": obj.object_name,
"applied": apply,
}
if apply:
# Taking the reservation lock fences an expired API before deletion.
if reservation:
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s",
(reservation["id"],),
)
store.delete(key=obj.object_name)
for job in jobs:
if (
job["state"] == "failed"
and job["source_key"] == obj.object_name
):
conn.execute(
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
(job["id"],),
)
conn.execute(
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
(job["attachment_id"], job["pipeline_version"]),
)
actions.append(action)
return {"apply": apply, "retention_seconds": grace, "actions": actions}
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--loop", action="store_true")
a = p.parse_args()
db, store = database(), ObjectStore()
try:
while True:
try:
print(
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
)
except Exception:
if not a.loop:
raise
print(
json.dumps(
{
"event": "cleanup_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
if not a.loop:
break
time.sleep(60)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+296
View File
@@ -0,0 +1,296 @@
import hashlib
import json
from uuid import UUID
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from src.cache import UserCache
from src.database import Database
from src.domain import Chat, Key, Message, User
from src.errors import DomainError, invalid, not_found
MESSAGE_COLUMNS = (
"m.*, ARRAY(SELECT attachment_id FROM message_attachments ma "
"WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids"
)
USER_COLUMNS = "id, username, display_name, created_at"
def user_dto(row) -> User:
return User(
row["id"], row["username"], row["display_name"], row["created_at"]
)
def chat_dto(row) -> Chat:
return Chat(
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
)
def message_dto(row) -> Message:
return Message(
row["id"],
row["chat_id"],
row["sender_id"],
row["text"],
row["created_at"],
tuple(row.get("attachment_ids", ())),
)
class PostgresRepository:
def __init__(self, *, database: Database, cache: UserCache):
self.database = database
self.cache = cache
def add_user(self, *, user: User, password_hash: str) -> User:
try:
with self.database.connection() as conn:
conn.execute(
"INSERT INTO users(id,username,display_name,"
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
(
user.id,
user.username,
user.display_name,
password_hash,
user.created_at,
),
)
except UniqueViolation:
raise DomainError(
409, "conflict", "Username already exists"
) from None
return user
def get_user(self, *, user_id: UUID) -> User | None:
cached = self.cache.get(user_id=user_id)
if cached is not None:
return cached
with self.database.connection() as conn:
row = conn.execute(
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(user_id,),
).fetchone()
if row is None:
return None
user = user_dto(row)
self.cache.put(user=user)
return user
def credentials(self, *, username: str):
with self.database.connection() as conn:
row = conn.execute(
"SELECT "
+ USER_COLUMNS
+ ", password_hash FROM users WHERE username=%s",
(username,),
).fetchone()
if row is None:
return None, None
return user_dto(row), row["password_hash"]
def username_exists(self, *, username: str) -> bool:
with self.database.connection() as conn:
return (
conn.execute(
"SELECT 1 FROM users WHERE username=%s", (username,)
).fetchone()
is not None
)
def add_session(
self, *, digest: bytes, user_id: UUID, created_at, expires_at
):
with self.database.connection() as conn:
conn.execute(
"INSERT INTO sessions(token_digest,user_id,created_at,"
"expires_at) VALUES(%s,%s,%s,%s)",
(digest, user_id, created_at, expires_at),
)
def session_user(self, *, digest: bytes) -> UUID | None:
with self.database.connection() as conn:
row = conn.execute(
"SELECT user_id FROM sessions WHERE token_digest=%s "
"AND expires_at>clock_timestamp()",
(digest,),
).fetchone()
return row["user_id"] if row else None
def delete_session(self, *, digest: bytes):
with self.database.connection() as conn:
conn.execute(
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
)
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
members = chat.member_ids
if actor_id not in members or len(set(members)) != len(members):
raise invalid()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute(
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
(chat.id, chat.title, chat.created_at),
)
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO members(chat_id,user_id,position) "
"VALUES(%s,%s,%s)",
[
(chat.id, user_id, position)
for position, user_id in enumerate(members)
],
)
except ForeignKeyViolation:
raise invalid() from None
return chat
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
with self.database.connection() as conn:
row = conn.execute(
"SELECT c.id,c.title,c.created_at,"
"array_agg(m.user_id ORDER BY m.position) member_ids "
"FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
(chat_id, actor_id),
).fetchone()
if row is None:
raise not_found()
return chat_dto(row)
def add_message(
self, *, message: Message, idempotency_key: str
) -> Message:
# Область включает пользователя, метод и нормализованный путь чата.
scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}"
lock = int.from_bytes(
hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True
)
fingerprint = hashlib.sha256(
json.dumps(
{
"text": message.text,
"attachment_ids": [str(x) for x in message.attachment_ids],
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode()
).digest()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
prior = conn.execute(
"SELECT i.body_hash,i.message_id,"
"i.expires_at>clock_timestamp() active "
"FROM idempotency i WHERE user_id=%s AND chat_id=%s "
"AND key=%s FOR UPDATE",
(message.sender_id, message.chat_id, idempotency_key),
).fetchone()
if prior and prior["active"]:
if bytes(prior["body_hash"]) != fingerprint:
raise DomainError(409, "conflict", "Key already used")
row = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.id=%s",
(prior["message_id"],),
).fetchone()
return message_dto(row)
if message.attachment_ids:
rows = conn.execute(
"SELECT id,state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s) FOR SHARE",
(message.chat_id, list(message.attachment_ids)),
).fetchall()
if len(rows) != len(message.attachment_ids):
raise not_found()
if any(row["state"] == "failed" for row in rows):
raise DomainError(
409, "conflict", "Attachment is not ready"
)
conn.execute(
"INSERT INTO messages(id,chat_id,sender_id,"
"text,created_at) "
"VALUES(%s,%s,%s,%s,%s)",
(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
),
)
if message.attachment_ids:
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO message_attachments("
"message_id,chat_id,"
"attachment_id,position) "
"VALUES(%s,%s,%s,%s)",
[
(message.id, message.chat_id, aid, i)
for i, aid in enumerate(message.attachment_ids)
],
)
conn.execute(
"INSERT INTO idempotency(user_id,chat_id,key,body_hash,"
"message_id,expires_at) VALUES(%s,%s,%s,%s,%s,"
"clock_timestamp()+interval '24 hours') "
"ON CONFLICT(user_id,chat_id,key) DO UPDATE SET "
"body_hash=excluded.body_hash,message_id=excluded.message_id,"
"expires_at=excluded.expires_at",
(
message.sender_id,
message.chat_id,
idempotency_key,
fingerprint,
message.id,
),
)
except ForeignKeyViolation:
raise not_found() from None
return message
def list_chats(
self, *, actor_id: UUID, after: Key | None, limit: int
) -> list[Chat]:
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
parameters = (
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
"FROM members WHERE chat_id=c.id "
"ORDER BY position) member_ids "
"FROM chats c JOIN members own ON own.chat_id=c.id "
"WHERE own.user_id=%s "
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
parameters,
).fetchall()
return [chat_dto(row) for row in rows]
def list_messages(
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
) -> list[Message]:
condition = "AND (m.created_at,m.id)>(%s,%s)" if after else ""
parameters = (
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s "
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
parameters,
).fetchall()
return [message_dto(row) for row in rows]
+65
View File
@@ -0,0 +1,65 @@
import re
from typing import Annotated
from uuid import UUID
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
StrictStr,
ValidationInfo,
field_validator,
model_validator,
)
from src.cursor import parse_uuid
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
class Input(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value: object, info: ValidationInfo) -> object:
if isinstance(value, str) and (
(not value.strip() and info.field_name != "text")
or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
):
raise ValueError("Invalid text")
return value
class CreateUser(Input):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
class CreateMessage(Input):
text: StrictStr = Field(max_length=4000)
attachment_ids: list[UuidIn] = Field(
default_factory=list, max_length=10, strict=True
)
@model_validator(mode="after")
def content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Duplicate attachment")
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+192
View File
@@ -0,0 +1,192 @@
import base64
import hashlib
import re
import secrets
import threading
from datetime import timedelta
from uuid import UUID, uuid4
from src.cursor import decode_cursor, encode_cursor
from src.domain import Chat, Message, Record, User, order_key, utcnow
from src.errors import DomainError, invalid, not_found
from src.password import Passwords
from src.repository import PostgresRepository
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
items = rows[:limit]
cursor = (
encode_cursor(scope=scope, key=order_key(items[-1]))
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": cursor}
def unauthorized(*, basic=False):
challenge = (
'Basic realm="messenger", charset="UTF-8"'
if basic
else 'Bearer realm="messenger"'
)
return DomainError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": challenge},
)
def bearer(value):
if not value:
raise unauthorized()
scheme, _, token = value.partition(" ")
if scheme.lower() != "bearer" or not re.fullmatch(
r"[A-Za-z0-9_-]{32,256}", token
):
raise unauthorized()
return token
def token_digest(value):
return hashlib.sha256(value.encode()).digest()
class MessengerService:
def __init__(
self,
*,
repository: PostgresRepository,
passwords: Passwords,
session_ttl: int,
) -> None:
self.repository = repository
self.passwords = passwords
self.session_ttl = session_ttl
self.registration_locks = [threading.Lock() for _ in range(64)]
def identify(self, *, value: str | None) -> User:
token = bearer(value)
user_id = self.repository.session_user(digest=token_digest(token))
user = self.repository.get_user(user_id=user_id) if user_id else None
if user is None:
raise unauthorized()
return user
def register(
self, *, username: str, display_name: str, password: str
) -> User:
# Bound lock memory and avoid running Argon2 for every local duplicate.
# Different replicas still race safely through PostgreSQL UNIQUE.
stripe = hashlib.sha256(username.encode()).digest()[0] % 64
lock = self.registration_locks[stripe]
if not lock.acquire(timeout=8):
raise DomainError(503, "unavailable", "Registration busy")
try:
if self.repository.username_exists(username=username):
raise DomainError(409, "conflict", "Username already exists")
encoded = self.passwords.hash(password=password)
return self.repository.add_user(
user=User(uuid4(), username, display_name, utcnow()),
password_hash=encoded,
)
finally:
lock.release()
def login(self, *, authorization: str | None):
try:
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic":
raise ValueError
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
username, separator, password = plain.partition(":")
if (
not separator
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
or not 12 <= len(password) <= 128
or not password.strip()
):
raise ValueError
except ValueError, UnicodeError:
raise unauthorized(basic=True) from None
user, password_hash = self.repository.credentials(username=username)
if not self.passwords.verify(password=password, encoded=password_hash):
raise unauthorized(basic=True)
now = utcnow()
expires = now + timedelta(seconds=self.session_ttl)
token = secrets.token_urlsafe(32)
self.repository.add_session(
digest=token_digest(token),
user_id=user.id,
created_at=now,
expires_at=expires,
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires,
"user": user,
}
def logout(self, *, authorization: str | None):
self.identify(value=authorization)
self.repository.delete_session(
digest=token_digest(bearer(authorization))
)
def create_chat(
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
) -> Chat:
return self.repository.add_chat(
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
actor_id=actor_id,
)
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
def send(
self,
*,
actor_id: UUID,
chat_id: UUID,
text: str,
idempotency_key: str | None,
attachment_ids: list[UUID],
) -> Message:
if not idempotency_key or not re.fullmatch(
r"[A-Za-z0-9._:-]{1,128}", idempotency_key
):
raise invalid()
return self.repository.add_message(
message=Message(
uuid4(),
chat_id,
actor_id,
text,
utcnow(),
tuple(attachment_ids),
),
idempotency_key=idempotency_key,
)
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
scope = f"chats:{actor_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_chats(
actor_id=actor_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
def history(
self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None
) -> dict:
if actor_id not in chat.member_ids:
raise not_found()
chat_id = chat.id
scope = f"messages:{chat_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_messages(
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
+37
View File
@@ -0,0 +1,37 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str) -> str:
file = os.environ.get(name + "_FILE")
if file:
return Path(file).read_text().strip()
value = os.environ.get(name)
if not value:
raise ValueError("Missing required secret setting: " + name)
return value
def integer(name: str, default: int, low: int, high: int) -> int:
value = int(os.environ.get(name, str(default)))
if not low <= value <= high:
raise ValueError("Invalid setting: " + name)
return value
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 10
@classmethod
def load(cls):
return cls(
secret("DATABASE_URL"),
secret("CACHE_URL"),
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
integer("DB_POOL_SIZE", 10, 1, 32),
)
+453
View File
@@ -0,0 +1,453 @@
"""Two bounded consumers; every decode runs in a killable isolated process."""
import hashlib
import io
import json
import multiprocessing
import resource
import threading
import time
from pathlib import Path
from uuid import uuid4
import redis
from PIL import Image, ImageOps, UnidentifiedImageError
from src.images import MAX_BYTES, MAX_PIXELS
from src.object_store import ObjectStore
from src.pipeline import (
GROUP,
STREAM,
PermanentFailure,
broker,
database,
event,
fault,
retry,
validate_job,
)
from src.settings import integer
def sizes(width, height, bound):
longest = max(width, height)
return (
(width, height)
if longest <= bound
else (
max(1, width * bound // longest),
max(1, height * bound // longest),
)
)
def make_variants(raw, row):
if (
len(raw) != row["size_bytes"]
or hashlib.sha256(raw).hexdigest() != row["sha256"]
):
raise PermanentFailure("source_checksum_mismatch")
try:
with Image.open(io.BytesIO(raw)) as source:
if source.width * source.height > MAX_PIXELS:
raise PermanentFailure("pixel_limit")
if source.format not in {"PNG", "JPEG"}:
raise PermanentFailure("invalid_image")
source.load()
oriented = ImageOps.exif_transpose(source)
rgba = oriented.convert("RGBA")
rgb = Image.new("RGB", rgba.size, "white")
rgb.paste(rgba, mask=rgba.getchannel("A"))
rgb.info.clear()
yield "original", raw, row["content_type"], source.size
thumbnail = rgb.resize(
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
)
crop = ImageOps.fit(
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
optimized = rgb.resize(
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
)
for name, image, fmt, mime, options in [
(
"thumbnail",
thumbnail,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"crop",
crop,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"optimized",
optimized,
"WEBP",
"image/webp",
{
"quality": 82 if row["pipeline_version"] == 1 else 76,
"method": 4,
},
),
]:
output = io.BytesIO()
image.info.clear()
image.save(output, format=fmt, **options)
yield name, output.getvalue(), mime, image.size
except (
UnidentifiedImageError,
OSError,
ValueError,
Image.DecompressionBombError,
) as exc:
raise PermanentFailure("invalid_image") from exc
def process(job_id, fence, sender):
db = store = None
started = time.monotonic()
cpu = time.process_time()
try:
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
db, store = database(), ObjectStore()
with db.connection() as conn:
row = conn.execute(
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
(job_id, fence),
).fetchone()
if not row:
sender.send({"result": "stale"})
return
# Record limits before fault hooks: a slow external inspector must not
# consume the child timeout while attempting to observe the limit.
cgroup = Path("/sys/fs/cgroup")
limits = {
name: (cgroup / name).read_text().strip()
if (cgroup / name).exists()
else "unavailable"
for name in ["memory.max", "cpu.max"]
}
limits["address_space_bytes"] = list(
resource.getrlimit(resource.RLIMIT_AS)
)
with db.connection() as conn:
event(conn, job_id, "worker_resource_limits", limits)
fault(db, job_id, "before_effect")
upstream = store.get(key=row["source_key"])
try:
raw = upstream.read(MAX_BYTES + 1)
finally:
upstream.close()
upstream.release_conn()
results = []
for name, data, mime, (width, height) in make_variants(raw, row):
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
digest = hashlib.sha256(data).hexdigest()
store.put(
key=key,
stream=io.BytesIO(data),
size=len(data),
content_type=mime,
sha256=digest,
)
results.append((name, key, mime, len(data), digest, width, height))
fault(db, job_id, "after_effect")
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
allowed = conn.execute(
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
(row["attachment_id"],),
).fetchone()
valid_lease = conn.execute(
"SELECT %s::timestamptz>clock_timestamp() live",
(current["lease_until"],),
).fetchone()["live"]
if (
current["state"] != "processing"
or current["fence"] != fence
or not valid_lease
or allowed["pipeline_version"] != row["pipeline_version"]
):
event(
conn,
job_id,
"stale_completion_rejected",
{"fence": fence, "version": row["pipeline_version"]},
)
sender.send({"result": "stale"})
return
for result in results:
conn.execute(
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
(row["attachment_id"], row["pipeline_version"], *result),
)
conn.execute(
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
(row["attachment_id"], row["pipeline_version"], job_id, fence),
)
conn.execute(
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(results[0][1], row["attachment_id"]),
)
conn.execute(
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
event(
conn,
job_id,
"effect_committed",
{
"fence": fence,
"version": row["pipeline_version"],
"variants": 4,
},
)
fault(db, job_id, "after_commit")
# Cleanup is independently retryable; it cannot undo the committed effect.
if row["source_key"].startswith("objects/staging/"):
try:
store.delete(key=row["source_key"])
except Exception:
pass
sender.send(
{
"result": "ready",
"seconds": time.monotonic() - started,
"cpu_seconds": time.process_time() - cpu,
"rss_peak_mib": resource.getrusage(
resource.RUSAGE_SELF
).ru_maxrss
/ 1024,
}
)
except PermanentFailure as exc:
sender.send({"result": "permanent", "code": str(exc)})
except MemoryError:
sender.send({"result": "permanent", "code": "memory_limit"})
except Exception:
sender.send({"result": "transient", "code": "processing_unavailable"})
finally:
if store:
store.close()
if db:
db.close()
sender.close()
def claim(db, data, consumer):
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
).fetchone()
if (
not row
or str(row["attachment_id"]) != data["attachment_id"]
or row["pipeline_version"] != data["pipeline_version"]
):
return "obsolete", None
if row["state"] in {"ready", "failed", "obsolete"}:
return row["state"], row
if data["attempt"] < row["attempt"] or (
row["state"] == "queued" and data["attempt"] <= row["attempt"]
):
return "obsolete", row
if row["state"] == "processing":
return "busy", row
if data["attempt"] != row["attempt"] + 1:
return "obsolete", row
due = conn.execute(
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
).fetchone()["due"]
if not due:
return "busy", row
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
updated = conn.execute(
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
(consumer, lease, row["id"]),
).fetchone()
conn.execute(
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
)
event(
conn,
row["id"],
"claimed",
{
"attempt": updated["attempt"],
"fence": updated["fence"],
"version": updated["pipeline_version"],
},
)
return "claimed", updated
def ready_effect(db, store, row):
with db.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
).fetchall()
if len(variants) != 4:
return False
for variant in variants:
head = store.head(key=variant["object_key"])
if (
head is None
or head.size != variant["size_bytes"]
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
):
return False
return True
def consume(slot):
db, queue, store = database(), broker(), ObjectStore()
consumer = str(uuid4()) + f"-{slot}"
context = multiprocessing.get_context("spawn")
cursor = "0-0"
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
if lease <= timeout:
raise ValueError("Lease must exceed child timeout")
try:
while True:
try:
try:
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
reclaimed = queue.xautoclaim(
STREAM,
GROUP,
consumer,
# Do not steal an entry from a consumer inside its live budget.
min_idle_time=lease * 1000,
start_id=cursor,
count=1,
)
cursor, messages = reclaimed[0], reclaimed[1]
if not messages:
batch = queue.xreadgroup(
GROUP, consumer, {STREAM: ">"}, count=1, block=500
)
messages = batch[0][1] if batch else []
for entry, fields in messages:
try:
data = validate_job(
json.loads(fields.get("job", "null"))
)
except ValueError, TypeError, KeyError:
# Malformed notifications cannot become jobs; preserve diagnostic count.
with db.connection() as conn:
event(conn, None, "invalid_envelope")
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
continue
state, row = claim(db, data, consumer)
if state == "busy":
continue
if state == "ready" and not ready_effect(db, store, row):
with db.connection() as conn:
event(conn, row["id"], "ready_effect_missing")
continue
if state in {"ready", "obsolete", "failed"}:
with db.connection() as conn:
event(
conn,
data["job_id"],
"duplicate_decision",
{"state": state, "entry": entry},
)
if state == "claimed":
receiver, sender = context.Pipe(duplex=False)
child = context.Process(
target=process,
args=(str(row["id"]), row["fence"], sender),
)
child.start()
sender.close()
child.join(timeout)
if child.is_alive():
child.kill()
child.join(2)
result = {
"result": "transient",
"code": "worker_timeout",
}
else:
try:
result = (
receiver.recv()
if receiver.poll()
else {
"result": "transient",
"code": "worker_crashed",
}
)
except EOFError:
result = {
"result": "transient",
"code": "worker_crashed",
}
receiver.close()
child.close()
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
(row["id"],),
).fetchone()
event(conn, row["id"], "child_finished", result)
if (
current["state"] == "processing"
and current["fence"] == row["fence"]
):
retry(
conn,
current,
result.get("code", "processing_failed"),
permanent=result["result"] == "permanent",
)
# Commit result or a retry/terminal decision precedes ack.
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
except Exception:
print(
json.dumps(
{
"event": "worker_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(0.5)
finally:
queue.close()
store.close()
db.close()
def main():
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
threads = [
threading.Thread(target=consume, args=(i,), daemon=False)
for i in range(count)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if __name__ == "__main__":
main()
+119
View File
@@ -0,0 +1,119 @@
import json
import os
from pathlib import Path
from uuid import uuid4
import httpx
import pytest
from jsonschema import Draft202012Validator, FormatChecker
CONTRACT = json.loads(
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
)
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8085")
PASSWORD = "Synthetic-Password-🌍-123"
TOKENS = {}
def check(result: httpx.Response, status: int, schema: str) -> dict:
assert result.status_code == status, result.text
assert result.headers["x-request-id"]
assert result.headers["x-instance-id"]
assert result.headers["content-type"].startswith("application/json")
payload = result.json()
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
payload
)
if schema == "Error":
assert payload["error"]["request_id"] == result.headers["x-request-id"]
assert (
payload["error"]["code"]
== {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
500: "internal_error",
503: "unavailable",
}[status]
)
return payload
def register(client: httpx.Client, **overrides) -> dict:
data = {
"username": "u_" + uuid4().hex[:24],
"display_name": "Алиса 🌍",
"password": PASSWORD,
}
data.update(overrides)
user = check(client.post("/api/v1/users", json=data), 201, "User")
session = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(data["username"], data["password"]),
),
201,
"Session",
)
TOKENS[user["id"]] = session["token"]
return user
@pytest.fixture
def client():
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as client:
yield client
@pytest.fixture
def actors(client):
return register(client), register(client), register(client)
@pytest.fixture
def chat(client, actors):
alice, bob, _ = actors
return check(
client.post(
"/api/v1/chats",
headers=headers(alice),
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
),
201,
"Chat",
)
def headers(actor: dict) -> dict:
return {
"Authorization": "Bearer " + TOKENS[actor["id"]],
"Idempotency-Key": uuid4().hex,
}
def history(client, actor, chat_id, limit=7):
found, cursor = [], None
while True:
params = {"limit": limit}
if cursor:
params["cursor"] = cursor
page = check(
client.get(
f"/api/v1/chats/{chat_id}/messages",
headers=headers(actor),
params=params,
),
200,
"MessagePage",
)
found.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return found
+412
View File
@@ -0,0 +1,412 @@
import base64
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from uuid import UUID, uuid4
import httpx
import pytest
from conftest import BASE_URL, PASSWORD, check, headers, history, register
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
def test_health(client, path):
assert check(client.get(path), 200, "Health") == {"status": "ok"}
@pytest.mark.parametrize(
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
)
def test_identification(client, value):
check(
client.get(
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
),
401,
"Error",
)
def test_identity(client, actors):
assert (
check(
client.get("/api/v1/users/me", headers=headers(actors[0])),
200,
"User",
)
== actors[0]
)
@pytest.mark.parametrize(
"data",
[
{},
[],
None,
{"username": "ABC", "display_name": "A"},
{"username": "abc\n", "display_name": "A"},
{"username": 123, "display_name": "A"},
{"username": "abc", "display_name": " "},
{"username": "abc", "display_name": "x" * 101},
{"username": "abc", "display_name": "A", "extra": True},
{"username": "abc", "display_name": "\ud800"},
],
)
def test_invalid_user(client, data):
if isinstance(data, dict):
data = {"password": PASSWORD, **data}
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
@pytest.mark.parametrize(
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
)
def test_malformed_json(client, raw):
check(client.post("/api/v1/users", content=raw), 400, "Error")
def test_auth_before_json(client):
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
def test_duplicate_username(client, actors):
check(
client.post(
"/api/v1/users",
json={
"username": actors[0]["username"],
"display_name": "B",
"password": PASSWORD,
},
),
409,
"Error",
)
def test_unicode_codepoints(client, actors, chat):
user = register(client, display_name="🌍" * 100)
assert user["display_name"] == "🌍" * 100
text = "🌍" * 4000
result = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": text},
),
201,
"Message",
)
assert result["text"] == text
assert result["sender_id"] == actors[0]["id"]
@pytest.mark.parametrize(
"method,suffix,body",
[
("GET", "", None),
("GET", "/messages", None),
("POST", "/messages", {"text": "hello"}),
],
)
def test_acl(client, actors, chat, method, suffix, body):
outsider = headers(actors[2])
actual = check(
client.request(
method,
f"/api/v1/chats/{chat['id']}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
missing = check(
client.request(
method,
f"/api/v1/chats/{uuid4()}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
assert actual["error"]["message"] == missing["error"]["message"]
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
def test_bad_message(client, actors, chat, value):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": value},
),
400,
"Error",
)
def test_sender_not_client_controlled(client, actors, chat):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "Hi", "sender_id": actors[1]["id"]},
),
400,
"Error",
)
@pytest.mark.parametrize(
"case",
[
"duplicate",
"unknown",
"no_caller",
"empty",
"wrong_type",
"bad_uuid",
"too_many",
"blank_title",
],
)
def test_invalid_chat(client, actors, case):
a, b, _ = actors
members = {
"duplicate": [a["id"], a["id"]],
"unknown": [a["id"], str(uuid4())],
"no_caller": [b["id"]],
"empty": [],
"wrong_type": a["id"],
"bad_uuid": ["broken"],
"too_many": [str(uuid4()) for _ in range(101)],
"blank_title": [a["id"]],
}[case]
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={
"title": " " if case == "blank_title" else "Title",
"member_ids": members,
},
),
400,
"Error",
)
@pytest.mark.parametrize(
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
)
def test_bad_limit(client, actors, chat, limit):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"limit": limit},
),
400,
"Error",
)
@pytest.mark.parametrize(
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
)
def test_bad_cursor(client, actors, chat, cursor):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"cursor": cursor},
),
400,
"Error",
)
def test_pagination(client, actors, chat):
a, b, outsider = actors
sent = []
path = f"/api/v1/chats/{chat['id']}/messages"
for i in range(23):
sent.append(
check(
client.post(
path,
headers=headers(a if i % 2 else b),
json={"text": f"Msg {i}"},
),
201,
"Message",
)
)
got = history(client, a, chat["id"])
assert got == sorted(
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
)
first = check(
client.get(path, headers=headers(a), params={"limit": 1}),
200,
"MessagePage",
)
second = check(
client.get(
path,
headers=headers(b),
params={"limit": 22, "cursor": first["next_cursor"]},
),
200,
"MessagePage",
)
assert second["items"] == got[1:]
assert second["next_cursor"] is None
chats = check(
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
)
assert chats["items"] == []
other = check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": "Other", "member_ids": [a["id"]]},
),
201,
"Chat",
)
check(
client.get(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(a),
params={"cursor": first["next_cursor"]},
),
400,
"Error",
)
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
data = json.loads(
base64.urlsafe_b64decode(
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
)
)
data["at"] = 17
forged = (
base64.urlsafe_b64encode(json.dumps(data).encode())
.decode()
.rstrip("=")
)
check(
client.get(path, headers=headers(a), params={"cursor": forged}),
400,
"Error",
)
def test_chat_pagination_scope(client, actors):
a, b, _ = actors
created = []
for i in range(4):
created.append(
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
),
201,
"Chat",
)
)
page = check(
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
200,
"ChatPage",
)
assert page["items"] == created[:2]
last = check(
client.get(
"/api/v1/chats",
headers=headers(a),
params={"limit": 2, "cursor": page["next_cursor"]},
),
200,
"ChatPage",
)
assert last["items"] == created[2:]
assert last["next_cursor"] is None
check(
client.get(
"/api/v1/chats",
headers=headers(b),
params={"cursor": page["next_cursor"]},
),
400,
"Error",
)
def test_concurrent_messages(client, actors, chat):
barrier = threading.Barrier(10)
a = actors[0]
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(i):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
return [
check(
conn.post(
path, headers=headers(a), json={"text": f"{i}:{j}"}
),
201,
"Message",
)["id"]
for j in range(10)
]
with ThreadPoolExecutor(max_workers=10) as pool:
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
rows = history(client, a, chat["id"])
assert len(rows) == len(ids) == len(set(ids)) == 100
assert set(ids) == {row["id"] for row in rows}
assert len({row["text"] for row in rows}) == 100
def test_registration_race(client):
username = "race_" + uuid4().hex[:20]
barrier = threading.Barrier(10)
def worker(_):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
result = conn.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Test",
"password": PASSWORD,
},
)
check(
result,
result.status_code,
"User" if result.status_code == 201 else "Error",
)
return result.status_code
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(worker, range(10)))
assert statuses.count(201) == 1
assert statuses.count(409) == 9
def test_unknown_path(client):
check(client.get("/does-not-exist"), 404, "Error")
+277
View File
@@ -0,0 +1,277 @@
import hashlib
import io
import time
from pathlib import Path
from uuid import uuid4
import pytest
from conftest import check, headers
from PIL import Image
PNG = (Path(__file__).parents[1] / "tests/fixtures/sample.png").read_bytes()
def upload(
client, actor, chat, *, data=PNG, filename="Фото 🌍.png", mime="image/png"
):
return client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**headers(actor), "Content-Type": mime},
params={"filename": filename},
content=data,
)
def wait_ready(client, actor, attachment):
end = time.monotonic() + 60
while time.monotonic() < end:
row = check(
client.get(
f"/api/v1/attachments/{attachment['id']}",
headers=headers(actor),
),
200,
"Attachment",
)
if row["status"] == "ready":
return row
assert row["status"] != "failed", row
time.sleep(0.2)
raise TimeoutError("Pipeline readiness")
def test_attachment_roundtrip_acl_and_message(client, actors, chat):
attachment = check(upload(client, actors[0], chat), 202, "Attachment")
assert attachment["status"] == "queued" and attachment["variants"] == []
attachment = wait_ready(client, actors[0], attachment)
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
assert attachment["size_bytes"] == len(PNG)
path = f"/api/v1/attachments/{attachment['id']}"
assert (
check(client.get(path, headers=headers(actors[1])), 200, "Attachment")
== attachment
)
downloaded = client.get(path + "/content", headers=headers(actors[1]))
assert downloaded.status_code == 200 and downloaded.content == PNG
assert downloaded.headers["Content-Type"] == "image/png"
assert not downloaded.history
for suffix in ["", "/content"]:
check(
client.get(path + suffix, headers=headers(actors[2])), 404, "Error"
)
check(client.get(path + suffix), 401, "Error")
thumb = client.get(
path + "/content",
headers=headers(actors[1]),
params={"variant": "thumbnail"},
)
assert thumb.status_code == 200
with Image.open(io.BytesIO(thumb.content)) as image:
assert image.format == "JPEG" and image.size == (256, 160)
sent = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[1]),
json={"text": "", "attachment_ids": [attachment["id"]]},
),
201,
"Message",
)
assert sent["attachment_ids"] == [attachment["id"]]
assert sent["sender_id"] == actors[1]["id"]
@pytest.mark.parametrize(
"name",
[
"../x.png",
"x/y.png",
"x\\y.png",
"x\x00.png",
"x\x7f.png",
"x\n.png",
"a" * 256,
"",
],
)
def test_filename_guard(client, actors, chat, name):
check(upload(client, actors[0], chat, filename=name), 400, "Error")
@pytest.mark.parametrize(
"data,mime,status",
[
(b"not an image", "image/png", 415),
(PNG, "image/jpeg", 415),
(PNG, "application/octet-stream", 415),
(PNG[:25], "image/png", 415),
(b"", "image/png", 415),
(b"x" * (10 * 1024 * 1024 + 1), "image/png", 413),
],
)
def test_image_guards(client, actors, chat, data, mime, status):
r = upload(client, actors[0], chat, data=data, mime=mime)
check(r, status, "Error")
def test_jpeg_original_unchanged(client, actors, chat):
stream = io.BytesIO()
Image.new("RGB", (320, 200), (30, 60, 90)).save(stream, format="JPEG")
jpeg = stream.getvalue()
attachment = check(
upload(
client,
actors[0],
chat,
data=jpeg,
filename="camera.jpg",
mime="image/jpeg",
),
202,
"Attachment",
)
attachment = wait_ready(client, actors[0], attachment)
r = client.get(
f"/api/v1/attachments/{attachment['id']}/content",
headers=headers(actors[1]),
)
assert r.content == jpeg
def test_cross_chat_link_and_unknown(client, actors, chat):
a = check(upload(client, actors[0], chat), 202, "Attachment")
other = check(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "Other", "member_ids": [actors[0]["id"]]},
),
201,
"Chat",
)
for identity in [a["id"], str(uuid4())]:
check(
client.post(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [identity]},
),
404,
"Error",
)
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [a["id"], a["id"]]},
),
400,
"Error",
)
def test_default_attachments_are_semantically_equal(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
first = check(
client.post(path, headers=h, json={"text": "Semantic default"}),
201,
"Message",
)
second = check(
client.post(
path,
headers=h,
json={"attachment_ids": [], "text": "Semantic default"},
),
201,
"Message",
)
assert first == second and first["attachment_ids"] == []
def test_pixel_limit_before_decode(client, actors, chat):
# Compressible synthetic image stays below10MiB but exceeds40M pixels.
stream = io.BytesIO()
Image.new("1", (8000, 5001)).save(stream, format="PNG")
assert len(stream.getvalue()) < 10 * 1024 * 1024
r = upload(client, actors[0], chat, data=stream.getvalue())
assert r.status_code == 413
def test_chunked_body_limit_without_content_length(client, actors, chat):
r = client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**headers(actors[0]), "Content-Type": "image/png"},
params={"filename": "chunked.png"},
content=iter([b"x" * (10 * 1024 * 1024 + 1)]),
)
assert r.request.headers.get("content-length") is None
assert r.status_code == 413
assert r.json()["error"]["code"] == "payload_too_large"
assert r.json()["error"]["request_id"] == r.headers["x-request-id"]
@pytest.mark.parametrize("size", [(1, 17), (320, 200), (1600, 900)])
def test_variant_geometry_and_alpha(client, actors, chat, size):
stream = io.BytesIO()
Image.new("RGBA", size, (0, 0, 0, 0)).save(stream, format="PNG")
a = check(
upload(client, actors[0], chat, data=stream.getvalue()),
202,
"Attachment",
)
a = wait_ready(client, actors[0], a)
for v in a["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=headers(actors[1]),
params={"variant": v["name"]},
)
assert len(r.content) == v["size_bytes"]
assert hashlib.sha256(r.content).hexdigest() == v["sha256"]
with Image.open(io.BytesIO(r.content)) as image:
assert image.size == (v["width"], v["height"])
assert not image.getexif()
assert min(image.convert("RGB").getpixel((0, 0))) >= 250
if v["name"] == "crop":
assert image.size == (128, 128)
else:
bound = 256 if v["name"] == "thumbnail" else 1280
assert max(image.size) <= bound
assert image.width <= size[0] and image.height <= size[1]
def test_exif_orientation_applied_but_original_preserved(client, actors, chat):
stream = io.BytesIO()
exif = Image.Exif()
exif[274] = 6
exif[270] = "Synthetic metadata must be removed"
Image.new("RGB", (320, 200), "red").save(stream, format="JPEG", exif=exif)
raw = stream.getvalue()
a = check(
upload(
client,
actors[0],
chat,
data=raw,
mime="image/jpeg",
filename="exif.jpg",
),
202,
"Attachment",
)
a = wait_ready(client, actors[0], a)
r = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=headers(actors[1])
)
assert r.content == raw
v = next(v for v in a["variants"] if v["name"] == "thumbnail")
assert (v["width"], v["height"]) == (160, 256)
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=headers(actors[1]),
params={"variant": "thumbnail"},
)
with Image.open(io.BytesIO(r.content)) as image:
assert image.size == (160, 256) and not image.getexif()
+26
View File
@@ -0,0 +1,26 @@
import json
from uuid import uuid4
import pytest
from conftest import check, headers
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
def test_only_utf8_json(client, encoding):
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
payload["password"] = "Synthetic-encoding-test-123"
result = client.post(
"/api/v1/users",
content=json.dumps(payload).encode(encoding),
headers={"Content-Type": "application/json"},
)
check(result, 400, "Error")
def test_leading_zero_limit(client, actors, chat):
result = client.get(
"/api/v1/chats/" + chat["id"] + "/messages",
headers=headers(actors[0]),
params={"limit": "0001"},
)
check(result, 200, "MessagePage")
+23
View File
@@ -0,0 +1,23 @@
from conftest import check, headers
def test_history_acl_precedes_query_validation(client, actors, chat):
path = f"/api/v1/chats/{chat['id']}/messages"
check(
client.get(
path,
headers=headers(actors[2]),
params={"limit": 0, "cursor": "broken"},
),
404,
"Error",
)
check(
client.get(
path,
headers=headers(actors[0]),
params={"limit": 0, "cursor": "broken"},
),
400,
"Error",
)
+75
View File
@@ -0,0 +1,75 @@
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import httpx
import pytest
from conftest import BASE_URL, check, headers, history
@pytest.mark.parametrize("key", [None, "", "a b", "x" * 129, "a/b"])
def test_key_required_and_valid(client, actors, chat, key):
h = headers(actors[0])
h.pop("Idempotency-Key")
if key is not None:
h["Idempotency-Key"] = key
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=h,
json={"text": "Key"},
),
400,
"Error",
)
def test_parallel_replay_across_instances(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
def send(_):
with httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15) as c:
r = c.post(path, headers=h, json={"text": "Один результат 🌍"})
return check(r, 201, "Message"), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({r[0]["id"] for r in results}) == 1
assert len({r[1] for r in results}) >= 2
assert history(client, actors[0], chat["id"]) == [results[0][0]]
check(
client.post(path, headers=h, json={"text": "Другой текст"}),
409,
"Error",
)
# Другой пользователь может независимо использовать тот же ключ.
other = headers(actors[1])
other["Idempotency-Key"] = h["Idempotency-Key"]
second = check(
client.post(path, headers=other, json={"text": "Боб"}), 201, "Message"
)
assert second["id"] != results[0][0]["id"]
def test_replay_rechecks_acl(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
check(
client.post(path, headers=h, json={"text": "Existing"}), 201, "Message"
)
outsider = headers(actors[2])
outsider["Idempotency-Key"] = h["Idempotency-Key"]
check(
client.post(path, headers=outsider, json={"text": "Existing"}),
404,
"Error",
)
check(
client.post(
path,
headers={"Idempotency-Key": uuid4().hex},
json={"text": "Existing"},
),
401,
"Error",
)
+114
View File
@@ -0,0 +1,114 @@
from uuid import uuid4
import httpx
import pytest
from conftest import PASSWORD, TOKENS, check, headers, register
@pytest.mark.parametrize(
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
)
def test_password_validation(client, password):
check(
client.post(
"/api/v1/users",
json={
"username": "pwd_" + uuid4().hex[:20],
"display_name": "Test",
"password": password,
},
),
400,
"Error",
)
def test_full_utf8_password(client):
password = "🌍" * 127 + ":"
user = register(client, password=password)
assert (
check(
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
)
== user
)
@pytest.mark.parametrize(
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
)
def test_bad_basic(client, authorization):
result = client.post(
"/api/v1/auth/sessions",
headers={"Authorization": authorization} if authorization else {},
)
check(result, 401, "Error")
assert (
result.headers["www-authenticate"]
== 'Basic realm="messenger", charset="UTF-8"'
)
def test_wrong_password_and_unknown_user(client):
user = register(client)
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
result = client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
)
data = check(result, 401, "Error")
assert data["error"]["message"] == "Authentication required"
def test_logout_is_not_revived_by_cache(client):
user = register(client)
auth = headers(user)
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
assert result.status_code == 204 and result.content == b""
assert result.headers["x-request-id"]
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
check(
client.delete("/api/v1/auth/sessions/current", headers=auth),
401,
"Error",
)
def test_only_current_session_revoked(client):
user = register(client)
another = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(user["username"], PASSWORD),
),
201,
"Session",
)
assert another["token"] != TOKENS[user["id"]]
assert len(another["token"]) >= 32
assert (
client.delete(
"/api/v1/auth/sessions/current", headers=headers(user)
).status_code
== 204
)
result = client.get(
"/api/v1/users/me",
headers={"Authorization": "Bearer " + another["token"]},
)
assert check(result, 200, "User") == user
def test_bearer_source_and_ignored_user_header(client, actors):
alice, bob, _ = actors
auth = {**headers(alice), "X-User-Id": bob["id"]}
assert (
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
== alice["id"]
)
result = client.get(
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
)
check(result, 401, "Error")
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'
+119
View File
@@ -0,0 +1,119 @@
import copy
from datetime import UTC, datetime
from uuid import uuid4
import pytest
from jsonschema import Draft202012Validator, FormatChecker
from src.pipeline import validate_job
def example():
return dict(
schema_version=1,
job_id=str(uuid4()),
type="attachment.process.v1",
attachment_id=str(uuid4()),
pipeline_version=1,
attempt=1,
created_at=datetime.now(UTC).isoformat(),
)
def test_schema_and_runtime_agree_on_valid_job():
import json
from pathlib import Path
schema = json.loads(Path("contracts/jobs.schema.json").read_text())
for version in [1, 2]:
data = example()
data["pipeline_version"] = version
data.update(
traceparent="00-" + "a" * 32 + "-" + "b" * 16 + "-01",
tracestate="vendor=opaque",
)
Draft202012Validator(schema, format_checker=FormatChecker()).validate(
data
)
assert validate_job(data) == data
@pytest.mark.parametrize(
"field,value",
[
("schema_version", True),
("schema_version", 2),
("type", "other"),
("job_id", 123),
("job_id", "broken"),
("attachment_id", None),
("pipeline_version", 0),
("pipeline_version", True),
("attempt", 0),
("attempt", 1.0),
("created_at", "2026-01-01"),
("created_at", "2026-01-01T12:00:00"),
("created_at", None),
("traceparent", "not-w3c"),
("tracestate", "x" * 513),
("url", "http://invalid.example"),
("bytes", "secret-contents"),
("filename", "../../bad"),
],
)
def test_invalid_notification_is_rejected(field, value):
data = copy.deepcopy(example())
data[field] = value
with pytest.raises((ValueError, TypeError)):
validate_job(data)
def test_worker_rejects_oversized_pixels_before_decode():
import hashlib
import io
from PIL import Image
from src.pipeline import PermanentFailure
from src.worker import make_variants
stream = io.BytesIO()
Image.new("1", (8000, 5001)).save(stream, format="PNG")
raw = stream.getvalue()
row = dict(
size_bytes=len(raw),
sha256=hashlib.sha256(raw).hexdigest(),
content_type="image/png",
pipeline_version=1,
)
with (
pytest.warns(Image.DecompressionBombWarning),
pytest.raises(PermanentFailure, match="pixel_limit"),
):
list(make_variants(raw, row))
def test_worker_rejects_changed_source_checksum():
from src.pipeline import PermanentFailure
from src.worker import make_variants
with pytest.raises(PermanentFailure, match="source_checksum_mismatch"):
list(make_variants(b"changed", dict(size_bytes=7, sha256="0" * 64)))
def test_worker_rejects_invalid_image_as_permanent():
import hashlib
from src.pipeline import PermanentFailure
from src.worker import make_variants
raw = b"not an image"
with pytest.raises(PermanentFailure, match="invalid_image"):
list(
make_variants(
raw,
dict(
size_bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()
),
)
)
Generated
+1095
View File
File diff suppressed because it is too large Load Diff