diff --git a/.env.example b/.env.example index d99d26a2..f6143d63 100644 --- a/.env.example +++ b/.env.example @@ -5,16 +5,62 @@ APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. +# The files are created by `python scripts/init_secrets.py` and mounted by compose as secrets. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`. +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url +# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas). API_REPLICAS=2 +# Idempotency-Key results are kept at least this long (contract: >= 24 h). IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path. +MESSAGE_WRITE_PATH=function +# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on +# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler. +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Contract limits (may be lowered, never raised above 10485760 / 40000000). MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once. +UPLOAD_CONCURRENCY=4 +UPLOAD_QUEUE_TIMEOUT_SECONDS=5 +UPLOAD_READ_TIMEOUT_SECONDS=30 +IMAGE_VERIFY_TIMEOUT_SECONDS=10 +IMAGE_VERIFY_CONCURRENCY=1 +JPEG_MAX_MEMORY_MB=256 +S3_TIMEOUT_SECONDS=3 +# Janitor: 'pending' intents older than this are failed uploads (object + row removed). +PENDING_GRACE_SECONDS=900 +JANITOR_INTERVAL_SECONDS=60 +# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty. +FAULT_POINTS= +# Additional settings of this implementation (defaults shown; all optional): +LOG_LEVEL=info +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_TIMEOUT_SECONDS=3 +CACHE_TIMEOUT_SECONDS=0.5 +PASSWORD_HASH_CONCURRENCY=2 +DOCS_ENABLED=true +# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests. +DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=8 +# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica. +API_CPUS=1.0 +API_MEMORY=512m +S3_CPUS=1.0 +S3_MEMORY=1g +PG_CPUS=2.0 +PG_MEMORY=1g +VALKEY_CPUS=0.5 +LB_CPUS=1.0 +PG_MAX_CONNECTIONS=100 +# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing) +APP_PUBLISH_PORT=8080 +LB_STATS_PORT=8404 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..3c10fe54 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# LF everywhere: identical files on Windows/Linux and inside `git archive | docker build -`. +* text=auto eol=lf +*.png binary +# cmd.exe expects CRLF in batch files. +*.cmd text eol=crlf diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..c838522c --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,38 @@ +name: Application (lab4) +on: [push, pull_request] +permissions: + contents: read +jobs: + app: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Synthetic secrets and env (CI only, never committed) + run: python scripts/init_secrets.py && cp .env.example .env + - name: Build, migrate once, bootstrap the S3 bucket, start 2 API replicas behind HAProxy + run: docker compose --env-file .env up -d --build --wait + - name: Contract and public smoke + run: make check test PYTHON=python + - name: Persistence across restart of LB, API replicas, PostgreSQL, Valkey and S3 + run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python + - name: Failover of the replica that served the test, then restore it + run: | + make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python + sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2 + - name: Reconciler (track 2) dry-run on the stand - read-only report + run: docker compose run --rm reconciler + - name: Own tests (unit + stand; every failure is restored in finally) + run: | + python -m pip install --require-hashes --no-deps -r requirements.txt + python -m pip install -r requirements-test.txt + make student-test PYTHON=python + - name: Logs + if: always() + run: docker compose logs --tail 100 api lb s3 s3-init + - name: Remove CI stand and its synthetic data + if: always() + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9da48416 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +# syntax=docker/dockerfile:1 +# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml +# (services api, migrate, s3-init, reconciler: one image, different commands). +# +# Python is interpreted, so there is no separate compile stage; the image +# contains only the interpreter, locked dependencies and the source code. +# Base image is pinned by version AND digest (python 3.13.16, Debian 12 bookworm). +FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +# 1. Dependencies first (layer cache survives source edits). Every package, +# including transitive ones, is pinned with sha256 hashes in requirements.txt; +# --no-deps guarantees nothing outside the lock file is installed. +COPY requirements.txt ./ +RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt + +# 2. JPEG integrity check: libjpeg-turbo's djpeg (strict mode, see src/messenger/images.py). +# Version pinned to the Debian 12 package; apt lists are removed in the same layer. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +# 3. Unprivileged runtime user. Code stays owned by root, so the process +# cannot modify its own code. +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +COPY src/ ./src/ +# Versioned SQL migrations, applied by `python -m messenger.migrate` (compose service migrate). +COPY migrations/ ./migrations/ +# Course contract, served read-only at /openapi.json for Swagger UI (/docs). +COPY contracts/openapi.json ./contracts/openapi.json + +# Non-secret defaults; compose.yaml passes the rest. Passwords come only as files +# (DATABASE_URL_FILE / CACHE_URL_FILE -> Docker secrets), never as ENV or COPY. +ENV PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + LOG_LEVEL=info \ + SHUTDOWN_TIMEOUT_SECONDS=8 \ + DOCS_ENABLED=true \ + SESSION_TTL_SECONDS=3600 \ + DATABASE_URL_FILE=/run/secrets/database_url + +# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out +# image distinguishable in `docker inspect` and in the "starting" log line of each replica. +ARG BUILD_REVISION=dev +LABEL org.opencontainers.image.revision=$BUILD_REVISION +ENV APP_REVISION=$BUILD_REVISION + +USER app:app +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-m", "messenger.healthcheck"] + +# Exec form: the Python server is PID 1 and receives SIGTERM directly (no shell in between). +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 5d86d171..3e68a6d2 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,70 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +# --------------------------------------------------------------------------- +# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public +# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash). +.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile + +secrets: + $(PYTHON) scripts/init_secrets.py + +# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy. +up: secrets + docker compose up -d --build --wait + +# Safe stop: containers stop, named volumes (data) stay. +down: + docker compose down + +logs: + docker compose logs --tail 50 api lb + +ps: + docker compose ps -a + +migrate: + docker compose run --rm migrate + +# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id. +lb-status: + $(PYTHON) scripts/lbctl.py status + $(PYTHON) scripts/lbctl.py replicas + +# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP. +restore: + sh scripts/restore-api.sh + $(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2} + +# Replace every API replica one by one with a freshly built image (no downtime). +rolling-update: + sh scripts/rolling-update.sh + +# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/). +failover-series: + $(PYTHON) bench/failover_series.py --output evidence/failover + +# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv). +student-test: + $(PYTHON) -m pytest student-tests -v + +unit-test: + $(PYTHON) -m pytest student-tests -m "not stack" -q + +explain: + $(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt + +# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only). +reconcile: + docker compose run --rm reconciler + +reconcile-apply: + docker compose run --rm reconciler --apply + +# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/). +bench-memory: + $(PYTHON) bench/upload_memory.py --output evidence/memory + +# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/). +bench-reconcile: + $(PYTHON) bench/reconcile_cost.py --output evidence/reconcile diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..c82b15ed 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,256 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Боданюк Алексей, БВТ2302 +- Целевая оценка: **5**, трек 2 «Согласованность двух хранилищ» +- Ветка: `solution/bvt2302-bodanyuk`. Форк: `https://gitea.privcloud.tech/wfa/mtusi2026lab4`. PR: «ЛР 4 — БВТ2302 — Боданюк — на 5» в mtusi2026/mtusi2026lab4 +- Стек: Python 3.13.16, Starlette 1.7.0, uvicorn 0.54.0, psycopg 3.3.6 + psycopg-pool 3.3.3, redis-py 8.1.0, argon2-cffi 25.1.0, **aiobotocore 3.9.2** (S3), **libjpeg-turbo 2.1.5 `djpeg`** (проверка JPEG); PostgreSQL 18.6, Valkey 8.1.3, HAProxy 3.2.26, **SeaweedFS 4.46**. Все образы закреплены по тегу **и** digest, все Python-пакеты — по sha256 ([requirements.txt](requirements.txt)), Debian-пакет — по версии ([Dockerfile](Dockerfile)) +- Источники: README, COURSE, контракт ([openapi.json](contracts/openapi.json), [семантика](contracts/README.md)), документация S3 (PutObject/Content-MD5, ListObjectsV2), SeaweedFS (`weed server`, s3.json identities), PNG (RFC 2083 / W3C PNG 3rd ed.: chunks, CRC, zlib, фильтры, Adam7), libjpeg-turbo (`djpeg -strict`, `-scale`, `-maxmemory`), HAProxy 3.2 (`nbsrv`, `use_backend`, `http-request set-timeout`), cgroup v2 (`memory.peak`, `memory.events`). Учтены все замечания преподавателя к PR lab4 других студентов ([раздел ниже](#проверки-на-которых-ловили-другие-работы)) +- Помощь ИИ: код, тесты, скрипты экспериментов и черновик отчёта написаны с помощью ИИ-ассистента Claude Code (Anthropic). Все команды и замеры реально выполнены на стенде ниже; вывод — в [evidence/](evidence/). Ответы на вопросы защиты — [docs/DEFENSE.md](docs/DEFENSE.md), согласованность хранилищ — [docs/CONSISTENCY.md](docs/CONSISTENCY.md), запуск на Windows и сценарий защиты — [docs/WINDOWS.md](docs/WINDOWS.md) ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Docker с Compose v2 (проверено: Docker 29.8.0, Compose 5.5.1), Python 3.9+ и GNU make (на Windows — см. [WINDOWS.md](docs/WINDOWS.md)). Учётные данные в Git не хранятся, их создаёт скрипт. + +```sh +git clone https://gitea.privcloud.tech/wfa/mtusi2026lab4 mtusi2026lab4 && cd mtusi2026lab4 && git checkout solution/bvt2302-bodanyuk +python3 scripts/init_secrets.py # пароли PostgreSQL/Valkey и ТРИ пары ключей S3 -> ./secrets (git-ignored), идемпотентно +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +# postgres, valkey, s3 healthy -> migrate (0001..0004) и s3-init (bucket + проверка прав) exit 0 -> api x2 healthy -> lb +docker compose logs s3-init # anonymous GET/LIST -> 403, права ключей api/reconciler -> OK +make check && make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh && make restore +make reconcile # трек 2: dry-run, read-only отчёт JSON +``` + +- **Windows:** публичным тестам нужен `PYTHONUTF8=1`; хуки — через `scripts/restart.cmd` и `scripts/stop-one.cmd` (тот же `.sh` в Git Bash). +- **Свои тесты:** `python3 -m venv .venv`, `pip install --require-hashes --no-deps -r requirements.txt`, `pip install -r requirements-test.txt`, затем `make student-test` (~25 мин, стенд должен быть поднят). Без стенда — `make unit-test`. +- **Все доказательства одной командой** (на чистом clone, ~70 мин): `python scripts/collect_evidence.py`. +- **Bootstrap S3** — одноразовый сервис `s3-init` (`python -m messenger.s3_bootstrap`), идемпотентный: существующий bucket не трогает, sentinel-объект перезаписывает, затем проверяет модель доступа и **валит стенд** (exit 1), если bucket не приватный или у ключей лишние права. Повтор: `docker compose run --rm s3-init`. +- **Остановка:** безопасно — `docker compose down` (volumes `pgdata`, `valkeydata`, `s3data` сохраняются). Намеренное удаление учебных данных — `docker compose down -v` и `rm -rf secrets`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +клиент ──127.0.0.1:8080──> lb: HAProxy 3.2 ──edge──> api × 2 (Starlette, без состояния и без volume) + │ backend api: /health/ready (PG + Valkey + S3) │ + │ backend api_core: /health/ready/core (PG + Valkey) — только если ├─data (internal)──> postgres 18.6: users, chats, messages, attachments (метаданные) + │ в api нет ни одного UP (S3 упал у всех) │ valkey 8.1.3: сессии + └ upload: timeout server 60 s └─storage (internal)─> s3: SeaweedFS 4.46, bucket messenger-attachments (байты) +migrate ──data──> 0001…0004 s3-init ──storage──> bucket, sentinel, проверка прав reconciler (по требованию) ──data+storage──> сверка +``` + +| Роль | Service | Образ | Health / readiness | Volume | Сеть / порт | Credentials | CPU / RAM | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Точка входа | `lb` | `haproxy:3.2.26-alpine@sha256:254e6370…` | `haproxy -c` + stats; бэкенды — активные проверки | — | edge; `127.0.0.1:8080`, `[::1]:8080`, `127.0.0.1:8404` | нет | 1.0 / 128m | +| API × 2 | `api` | `mtusi-messenger:lab4` из исходников | `/health/ready` (PG, Valkey, S3) | — | edge + data + storage, только `expose 8080` | `database_url`, `cache_url`, **своя** пара S3 (Read+Write bucket) | 1.0 / 512m, без swap | +| Миграции | `migrate` | тот же образ | exit 0 | — | data | `database_url` | 0.5 / 256m | +| Bootstrap S3 | `s3-init` | тот же образ | exit 0 | — | storage | admin + api + reconciler (для проверки прав) | 0.5 / 256m | +| БД | `postgres` | `postgres:18.6-bookworm@sha256:afc7e2d4…` | `pg_isready` | `pgdata` | data | `postgres_password` | 2.0 / 1g | +| Сессии | `valkey` | `valkey/valkey:8.1.3-bookworm@sha256:fea8b3e6…` | `valkey-cli ping` | `valkeydata` | data | `valkey_password` | 0.5 / 256m | +| Объекты | `s3` | `chrislusf/seaweedfs:4.46@sha256:08d51613…` | `GET /healthz` | `s3data` | storage, `8333` (S3 API) — не опубликован | identities из secrets ([start.sh](infra/seaweedfs/start.sh)) | 1.0 / 1g | +| Сверка (трек 2) | `reconciler` (профиль `tools`) | тот же образ | — | — | data + storage | `database_url`, пара S3 reconciler (+List) | 1.0 / 256m | + +**Что перенесено из lab3** (отдельный коммит `d916d09`, без изменений): API, миграции 0001–0003, HAProxy без sticky с re-resolve DNS, общая идемпотентность, graceful shutdown, сессии в Valkey, все тесты lab3. **Что сделано в lab4:** + +- **Выбор S3: SeaweedFS 4.46.** Репозитория `minio/minio` в Docker Hub больше нет (проверено при выборе), SeaweedFS активно развивается и поддерживает identity с правами на bucket. Один контейнер (`weed server`: master + volume + filer + S3 gateway). Web-интерфейсы master/filer/volume и их gRPC слушают **только 127.0.0.1 внутри контейнера** (`-ip.bind`), в сеть `storage` выходит лишь S3 API (8333/18333). Сеть internal, на хосте ничего не опубликовано. Телеметрия выключена, встроенный IAM API выключен, автосоздание bucket выключено. +- **Права.** Три пары ключей (`init_secrets.py`): `admin` (создание bucket) — только у `s3` и `s3-init`; `messenger-api` — `Read`+`Write` **на один bucket**, без `List` и `Admin`; `messenger-reconciler` — плюс `List`. Анонимной identity нет, поэтому bucket приватный. `s3-init` проверяет всё это при каждом запуске, включая канареечный bucket: API не видит его в ListBuckets и не может читать из него (403). +- **Модель данных** ([0004_attachments.sql](migrations/0004_attachments.sql), [DATA_MODEL.md](docs/DATA_MODEL.md)): `attachments` (метаданные; `object_key = o/<128 случайных бит>` назначает сервер; `status pending|ready`; карантин `lost_at/lost_reason`), `message_attachments` (FK + обратный поиск), `messages.attachment_ids uuid[]` (порядок, чтение истории остаётся одним index range scan). Отправка сообщения — SQL-функция `post_message_v2`, тоже одна транзакция. +- **Загрузка** ([attachments.py](src/messenger/attachments.py), [images.py](src/messenger/images.py)): `auth → членство (404) → filename / Content-Type / Content-Length (400/415/413) → слот загрузки → тело кусками с дедлайном → формат и пиксели по первым байтам → полная проверка → pending → PUT → ready → 201`. Подробно о порядке и сбоях — [CONSISTENCY.md](docs/CONSISTENCY.md). +- **Проверка содержимого.** PNG — свой потоковый валидатор: сигнатура, CRC **каждого** chunk, порядок chunk'ов, zlib-поток IDAT распаковывается порциями по 1 MiB и сразу выбрасывается, его длина должна быть ровно `height × (1 + байт в строке)` (Adam7 — по проходам), байт фильтра каждой строки ≤ 4, IEND в конце и ничего после него. Растр в памяти не создаётся. JPEG — `djpeg -strict -scale 1/8 -maxmemory`: эталонный декодер в отдельном процессе декодирует весь файл, любое предупреждение libjpeg — ошибка. Pillow для этого не годится: JPEG без данных после SOS он «успешно» декодирует в серую картинку (проверено). MIME определяется по байтам; несовпадение с `Content-Type` — 415. +- **Лимиты до чтения/декодирования.** `Content-Length > 10 MiB` → 413 без чтения тела; без длины (chunked) — 413 при пересечении лимита. Пиксели — из IHDR/SOFn по первым байтам: 20000×10000 → 413 через 33 байта, ничего не декодируется. Тело — один буфер ровно `Content-Length` байт (без копий до S3 и djpeg). Одновременно на реплику: `UPLOAD_CONCURRENCY=4` тела (остальные ждут ≤ 5 с с байтами в TCP-буфере, затем 503 + `Retry-After`), `IMAGE_VERIFY_CONCURRENCY=1` полная проверка. Дедлайны: тело 30 с, проверка 10 с, S3 3 с (+ время на объём), в HAProxy для загрузки `timeout server 60s`. +- **Скачивание** — только через API, без redirect: членство → `GET` объекта → стриминг кусками по 64 KiB (память не зависит от размера). Заголовки: настоящий `Content-Type`, `Content-Length`, `Content-Disposition: attachment` (ASCII-fallback + RFC 5987), `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `ETag` = SHA-256. +- **Готовность и деградированный режим.** `/health/ready` включает S3 (загрузка и скачивание — обязательный синхронный путь): S3 недоступен → 503. S3 — общая зависимость, и если он упал у всех, HAProxy переключается на backend `api_core` (проверка `/health/ready/core`: PG + Valkey). Чаты и текст работают, запросы к вложениям API сам отвечает 503 за ≤ 3 с. Если S3 потеряла одна реплика, она DOWN в `api`, трафик идёт на другую. +- **Janitor** (в каждой реплике, раз в ~60 с, advisory-замок): `pending` старше `PENDING_GRACE_SECONDS=900` → под `FOR UPDATE SKIP LOCKED` удалить объект, затем строку. +- **Reconciler (трек 2)** — [reconcile.py](src/messenger/reconcile.py), раздел «Трек 2» ниже. +- **`FAULT_POINTS`** (`after_s3_put`, `after_ready_commit`) — внедрение отказа ровно между шагами записи, только для экспериментов. По умолчанию пусто; неизвестное значение — ошибка конфигурации. + +Readiness-критичные зависимости: PostgreSQL, Valkey и S3. LB в деградированном режиме обходится без S3. + +Соответствие `.env.example` реальной конфигурации ([config.py](src/messenger/config.py), [compose.yaml](compose.yaml)): + +| Переменная | Значение | Где используется | +| --- | --- | --- | +| `APP_HOST`, `APP_PORT`, `APP_ENV`, `SESSION_TTL_SECONDS`, `DATABASE_URL_FILE`, `SESSION_STORE`, `CACHE_URL_FILE`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS` | как в lab3 | без изменений | +| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | `http://s3:8333`, `us-east-1`, `messenger-attachments` | клиент S3 API, s3-init, reconciler; bucket создаёт s3-init | +| `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` | `/run/secrets/s3_access_key_id`, `…/s3_secret_access_key` | у API это **его** пара (secrets `s3_api_*` смонтированы под этими именами), у reconciler — его пара | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | `10485760`, `40000000` | лимиты контракта; можно уменьшить, увеличить нельзя (ConfigError) | +| `UPLOAD_CONCURRENCY`, `UPLOAD_QUEUE_TIMEOUT_SECONDS`, `UPLOAD_READ_TIMEOUT_SECONDS` | `4`, `5`, `30` | слоты загрузки на реплику, ожидание слота, дедлайн тела | +| `IMAGE_VERIFY_CONCURRENCY`, `IMAGE_VERIFY_TIMEOUT_SECONDS`, `JPEG_MAX_MEMORY_MB` | `1`, `10`, `256` | одновременные полные проверки, дедлайн проверки, `djpeg -maxmemory` | +| `S3_TIMEOUT_SECONDS` | `3` | дедлайн каждого вызова S3 (PUT — плюс время на объём) | +| `PENDING_GRACE_SECONDS`, `JANITOR_INTERVAL_SECONDS` | `900`, `60` | janitor незавершённых загрузок | +| `FAULT_POINTS` | пусто | внедрение отказов в экспериментах | +| `API_CPUS/MEMORY`, `S3_CPUS/MEMORY`, `PG_*`, `VALKEY_CPUS`, `LB_CPUS` | 1.0/512m, 1.0/1g, 2.0/1g, 0.5, 1.0 | лимиты контейнеров | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Стенд: Windows 10 22H2, AMD Ryzen 5 3400G (4 ядра / 8 потоков), 30 ГБ RAM, Docker Desktop 29.8.0 (WSL2), Python 3.14 на хосте, GNU make 4.4.1. Проверки выполнены 2026-10-25 **на стенде из чистого clone** (новая папка, `git clone`, пустые volumes, secrets созданы заново) одной командой `python scripts/collect_evidence.py`, коммит `f9f5ab9`. Этот прогон нашёл два дефекта (см. ниже). После исправления (коммиты `046e257`, `fc50362`, `6fb70de`) на том же стенде целиком повторены свои тесты, публичные `make test` / `test-persistence` / `test-failover` ×2 и замер стоимости обхода — [final-commit-public-checks.txt](evidence/final-commit-public-checks.txt). -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда / сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Чистый clone, secrets, миграции, bucket с нуля | `init_secrets.py`, `compose config --quiet`, `compose up -d --build --wait`, логи `migrate` и `s3-init` | 7 сервисов healthy / exit 0; `0001…0004: applied`; `bucket messenger-attachments: created`; образы по digest | [clean-clone.txt](evidence/clean-clone.txt) | +| Контракт | `make check`, `make validate` | PASS, PASS | [make-check-validate-test.txt](evidence/make-check-validate-test.txt) | +| **3:** публичный smoke (checksum, посторонний) | `make test` | `PASS: lab4 public API scenario` (и на финальном коммите) | там же, [final-commit-public-checks.txt](evidence/final-commit-public-checks.txt) | +| **3:** S3 в стенде; метаданные в PostgreSQL, байты в приватном bucket, не в БД и не на API | `test_metadata_reveals_no_storage_details`, `test_original_is_not_stored_on_api_containers`, политика compose | строка `ready` + объект `o/` того же размера в bucket; в ФС реплик нет файла такого размера; у `api` нет volume | [student-tests.txt](evidence/student-tests.txt) | +| **3:** 201 ready, оригинал побайтно, сообщение с `attachment_ids` читают участники | `test_png_and_jpeg_roundtrip_byte_for_byte`, `test_message_with_attachments_and_history`, `scripts/demo_attachments.py` | PNG и JPEG: SHA-256 и размер совпадают; `text=""` + файл → 201; история возвращает `attachment_ids` по порядку | там же | +| **3:** реальные PNG/JPEG ≤ 10 MiB и ≤ 40 Мпикс, MIME по содержимому, опасные filename, лимиты до чтения и декодирования | `test_size_limit_through_the_load_balancer`, `test_exactly_40_megapixels_is_accepted`, `test_pixel_limit_before_decoding`, `test_bad_content_is_415` (16 случаев), `test_bad_filename_is_400` (14) | ровно 10 485 760 байт через LB → 201, +1 байт → 413 за < 5 с, chunked сверх лимита → 413; 8000×5000 → 201; 20000×10000 / 8001×5000 / JPEG 400 Мпикс → 413 за < 2 с | там же | +| **3:** upload → остановка обслужившей реплики → download через другую | `test_upload_survives_failover_of_the_serving_replica`, `make test-failover` ×2 | после SIGKILL обслужившей 5 скачиваний с другой реплики — те же байты; публичный тест для `api-1` и `api-2`: 131 / 132 записи, **0 неудачных**, оригинал на месте | [test-failover.txt](evidence/test-failover.txt) | +| **3:** restart с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; новые PID и `StartedAt` у `lb`, `api-*`, `postgres`, `valkey`, **`s3`**; прежний токен, файл скачивается | [test-persistence.txt](evidence/test-persistence.txt) | +| **4:** тесты: большой файл, повреждённое изображение, подмена MIME, чужое `attachment_id`, файл другого чата; ошибки пользователя не дают 500 | `test_attachments.py` (56 тестов) | 413 / 415 / 400 / 404 по контракту; `Api.call` проверяет у **каждого** ответа формат ошибки, `X-Request-Id`, `X-Instance-Id`, отсутствие stack trace | [student-tests.txt](evidence/student-tests.txt) | +| **4:** прерывание загрузки и отказ между S3 и БД не оставляют вечных объектов; grace period | `test_consistency.py` (4), [CONSISTENCY.md](docs/CONSISTENCY.md) | обрыв и зависание тела — ничего не записано; `after_s3_put` → `pending` + объект, janitor убирает оба (grace 3 с в тесте, 900 с в стенде); `after_ready_commit` → ничего не удалено, файл скачивается; свежий `pending` не тронут | там же | +| **4:** RAM одной загрузки ограничена; ≥ 10 одновременных загрузок с пиком памяти | `bench/upload_memory.py` (16 и 32 клиента) | см. «Эксперименты»: пик +81…113 МБ и при 16, и при 32 клиентах (4 слота), лимит 512 МБ, 0 OOM процесса API | [memory/summary.json](evidence/memory/summary.json) | +| **4:** bootstrap идемпотентен, отдельные ограниченные ключи S3, консоль закрыта, проверка приватности вне API | логи `s3-init` (при каждом `up`), `test_bucket_is_private_outside_the_api`, `test_compose_policy_…` | anonymous GET/LIST → 403; ключ API: Read/Write да, List/CreateBucket/чужой bucket — нет; ключ reconciler: List да; повторный bootstrap — `exists, kept as is`; порты 9333/8888/8080 SeaweedFS недоступны из сети | [clean-clone.txt](evidence/clean-clone.txt), [student-tests.txt](evidence/student-tests.txt) | +| **4:** отказ S3 | `test_s3_outage_is_503_and_text_keeps_working[stop/pause]` | readiness 503 на всех репликах (core 200), upload/download 503 за < 8 с, метаданные 200, **текст работает** (деградированный режим LB); после восстановления файл цел | там же | +| **5:** трек 2 | `test_reconcile.py` (6), `bench/reconcile_faults.py`, `bench/reconcile_cost.py` | см. раздел «Трек 2» | [evidence/reconcile/](evidence/reconcile/) | +| Перенесённое из lab3 | все тесты lab3 (балансировка, идемпотентность, отказы БД/Valkey, explain) | проходят без ослабления; адаптированы только списки сервисов и миграций | [student-tests.txt](evidence/student-tests.txt) | +| Публичный генератор нагрузки | `tests/load.py --duration 30 --concurrency 4` | 5416 успешных, 0 ошибок, 180 req/s, p50/p95/p99 = 18 / 49 / 64 мс | [load.json](evidence/load.json) | +| Свои тесты | `python -m pytest student-tests -v` | **316 passed, 4 skipped** (113 unit + 203 против стенда, 21 мин); skipped — 4 unit-теста djpeg на хосте без libjpeg-turbo, те же случаи проходят на стенде | [student-tests.txt](evidence/student-tests.txt) | + +**Что нашёл прогон из чистого clone (`f9f5ab9`).** 314 из 316 тестов прошли, два упали: + +1. `test_s3_outage_is_503_and_text_keeps_working[stop]`. После stop/start S3 первое скачивание получало 503: в пуле aiobotocore оставались keep-alive соединения к старому процессу, а повторов не было. Исправлено одной повторной попыткой в том же дедлайне — все операции идемпотентны по ключу ([storage.py](src/messenger/storage.py), `046e257`). +2. `test_restart_does_not_extend_ttl`. Рестарт Valkey и обеих реплик (readiness теперь ждёт и S3) длился дольше 12-секундного TTL теста. TTL теста увеличен до 30 с. + +Ещё дефекты нашли замеры памяти до этого прогона: + +- семафор загрузок отпускался сразу после чтения тела, а буфер жил до конца S3 PUT. При 32 клиентах пик рос до +214 МБ при 4 слотах. Теперь слот держится до конца загрузки (`277d06f`); +- djpeg, убитый OOM-killer'ом, давал 415 вместо 503 — исправлено; +- PNG-валидатор копировал остаток IDAT на каждом шаге (`unconsumed_tail`) — исправлено. + +И одну ошибку reconciler нашёл замер стоимости на том же стенде: `--apply` **без** `--verify-checksums` видел совпадающий размер и снимал карантин `checksum_mismatch` как `restored_object`, хотя сумму в этом режиме проверить не может. Теперь такой карантин снимает только проход с проверкой сумм (`6fb70de`). Тест дополнен, 6/6 тестов reconciler и smoke повторены на этом коммите — [reconcile-fix-rerun.txt](evidence/reconcile-fix-rerun.txt). Первый прогон своих тестов с двумя падениями сохранён как есть: [student-tests-first-run-f9f5ab9.txt](evidence/student-tests-first-run-f9f5ab9.txt). + +## Проверки, на которых ловили другие работы + +Замечания преподавателя к PR lab4 (#1–#13) других студентов, проверенные на своём стенде: + +| Замечание к чужим PR | У нас | Где проверено | +| --- | --- | --- | +| PNG с корректным IHDR и мусорным IDAT (86 байт), обрезанный PNG, первые 33 байта fixture, повреждение после первых 64 KiB → 201 | 415: CRC каждого chunk, zlib до точного размера, IEND обязателен | `test_bad_content_is_415[png:*]`, unit `test_broken_png_is_415` | +| JPEG без данных после SOS → 201 | 415: `djpeg -strict` (Pillow такой файл принимает — проверено) | `test_bad_content_is_415[jpeg: no scan data after SOS]` | +| PNG 6–10 MiB → 413 HTML от nginx | HAProxy не ограничивает тело; ровно 10 485 760 байт → 201 через LB | `test_size_limit_through_the_load_balancer` | +| 20000×10000 → 500 (`DecompressionBombError`) | 413 по IHDR, ничего не декодируется; 40 000 000 px принимается, 40 005 000 → 413 | `test_pixel_limit_before_decoding`, `test_exactly_40_megapixels_is_accepted` | +| `img.load()` до проверки размера (пик 252 МБ на файле 131 КБ) | размер берётся из заголовка до чтения остального тела; растр PNG не создаётся вовсе | unit `test_png_verification_is_streaming_and_fast` (40 Мпикс < 5 с) | +| API получает admin-ключи MinIO и создаёт bucket | у API своя пара: CreateBucket → AccessDenied (проверяет `s3-init`) | [clean-clone.txt](evidence/clean-clone.txt), `test_compose_policy_…` | +| S3 остановлен: readiness 200 / 500 за 9 с, upload 500 / 504 | readiness 503, upload 503 за < 8 с (S3 3 с), без 500 | `test_s3_outage_…[stop/pause]` | +| filename с C1 (U+0085) или из пробелов `%20%20%20` → 201 | 400; также `/`, `\`, NUL, TAB, U+2028, bidi U+202E, `.`, `..`, невалидный UTF-8, повтор параметра | `test_bad_filename_is_400` (14) | +| `{"attachment_ids":[A]}`, `{"text":null,…}`, `{"text":"x","attachment_ids":null}` → 201 | 400 | `test_message_body_validation` (12) | +| COMMIT прошёл, ответ потерян → API удаляет объект, остаётся `ready` без байтов | в обработчике компенсации нет; janitor трогает только `pending` | `test_lost_answer_after_commit_never_deletes_the_object` | +| dry-run продолжает частичное удаление | dry-run read-only на уровне БД (`default_transaction_read_only`) и S3-обёртки; промежуточного состояния «deleting» нет | `test_detect_and_safely_fix_faults_on_both_sides` (отпечатки до/после) | +| тело накапливается целиком, рост RAM «не доказывает потоковость» | буфер ровно `Content-Length`, число буферов ограничено слотами; замер 16 и 32 клиентов | [memory/summary.json](evidence/memory/summary.json) | +| тест шлёт POST в чужой чат без `Idempotency-Key`; гонка считает 5xx конфликтом | все POST сообщений в своих тестах — с ключом; `Api.call` требует точный ожидаемый код | [conftest.py](student-tests/conftest.py) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Уровень 4: память при ≥ 10 одновременных загрузках -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Гипотеза:** память реплики ограничена `UPLOAD_CONCURRENCY × MAX_UPLOAD_BYTES` плюс одна полная проверка (`IMAGE_VERIFY_CONCURRENCY`) и не зависит от числа клиентов. + +**Методика** ([bench/upload_memory.py](bench/upload_memory.py)): + +- перед каждым сценарием реплики api пересоздаются: cgroup `memory.peak` начинается с простоя; +- N клиентов стартуют одновременно (barrier) через LB, у каждого свой файл; +- `memory.current` каждой реплики снимается каждые 50 мс внутри контейнера; затем читаются `memory.peak` (все процессы контейнера, включая djpeg), `VmHWM` процесса Python и `memory.events` (OOM-kill) всех контейнеров. + +Лимит api: 1 CPU, 512 МБ, swap выключен (`memswap_limit`). Файлы: PNG 9,42 МБ из шума (не сжимается), JPEG 8000×5000 progressive 4:4:4 (0,6 МБ, самый дорогой для декодера: ~240 МБ буферов коэффициентов). Один прогон на сценарий, коммит `f9f5ab9`, сырые данные — [summary.json](evidence/memory/summary.json). + +| Сценарий | Ответы | Пик реплик − простой (≈ 75 МБ) | Python VmHWM | OOM-kill | +| --- | --- | --- | --- | --- | +| PNG ×16, 4 слота (по умолчанию) | 16 × 201, p50 3,6 с | +81 / +89 МБ | 153 МБ | 0 | +| PNG ×**32**, 4 слота | 25 × 201, 7 × 503 «слишком много загрузок» (+Retry-After) | +80 / +113 МБ — **как при 16** | 144 / 181 МБ | 0 | +| PNG ×32, **16 слотов** | 24 × 201, 8 × 503 «хранилище недоступно» | +209 / +193 МБ | 286 / 267 МБ | 0 | +| JPEG 40 Мпикс ×16, verify=1 (по умолчанию) | 16 × 201 | +242 / +242 МБ (djpeg) | 108 МБ | 0 | +| JPEG 40 Мпикс ×16, **verify=4** | 2 × 201, 14 × 503 «не хватило ресурсов» | 512 / 512 МБ (лимит) | 111 / 92 МБ | 6 / 7 (djpeg) | + +**Выводы:** + +- Удвоение числа клиентов при тех же 4 слотах пик не меняет; лишние клиенты быстро получают 503 с `Retry-After`, а их байты ждут в TCP-буферах, а не в процессе. +- Повышение границы до 16 слотов поднимает пик в 2–2,5 раза и упирается уже в S3: SeaweedFS ограничивает одновременные загрузки, PUT не укладывается в дедлайн, клиент получает 503. Значит, 4 слота — разумная граница при 1 CPU. +- Полная проверка JPEG — самая дорогая часть. При одной проверке за раз пик +242 МБ укладывается в 512 МБ. При четырёх cgroup убивает djpeg; API это переживает, отвечает 503 (а не 415 — исправлено после первого замера) и не теряет ни процесс, ни другие запросы. +- S3 (SeaweedFS, лимит 1 ГБ, `GOMEMLIMIT=600MiB`): anon-память 386–412 МБ; `memory.current` до 1 ГБ — это page cache записанных томов. + +**Ограничения.** Один хост и один прогон на сценарий (разброс между прогонами в моих повторах — до ±25 МБ). Клиенты на том же хосте. Python не возвращает освобождённую память ОС мгновенно, поэтому VmHWM — верхняя оценка. + +### Отказы + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker compose stop s3` | readiness 503, вложения 503, текст работает | так и есть, загрузка 503 за < 8 с; HAProxy на `api_core` | сразу после старта S3 (с повтором соединения); потерь нет | +| `docker compose pause s3` | то же, ответ по дедлайну, а не зависание | 503 за ≤ 3 с + запас; readiness 503 | после unpause; потерь нет | +| сбой между PUT и COMMIT (`FAULT_POINTS=after_s3_put`) | 503, `pending` + объект, уборка после grace | так и есть, janitor убрал за ≤ 30 с при grace 3 с | мусор убран, `ready`-данные не затронуты | +| потерян ответ после COMMIT (`after_ready_commit`) | 503, но файл цел | `ready`, объект на месте, скачивается | ничего не удалено | +| клиент оборвал тело / застрял на середине | ничего не записано / 400 по дедлайну | 0 строк, 0 объектов; 400 с `X-Instance-Id` | — | +| SIGKILL реплики, обслужившей загрузку | другая реплика отдаёт файл | 5/5 скачиваний, те же байты | — | +| stop одной реплики (публичный тест) ×2 | ≤ 10 с, без потерь и дублей | 0 неудачных записей, ~6 с до первой записи после хука (время самого `docker stop` с drain) | — | +| restart всего стенда с volumes | данные, сессия и оригинал на месте | PASS, все 6 долгоживущих контейнеров с новым PID | — | + +## Трек 2: согласованность двух хранилищ + +**Гипотеза.** Один повторяемый проход, который merge-join'ит упорядоченный листинг S3 и keyset-страницы `attachments` по `object_key`, находит все расхождения обеих сторон. Он безопасно исправляет их (dry-run ничего не меняет; свежие и используемые вложения не удаляются; повторный запуск идемпотентен), а стоит O(N) по запросам и O(1) по памяти. Устройство и правила — [CONSISTENCY.md](docs/CONSISTENCY.md), код — [reconcile.py](src/messenger/reconcile.py). + +**Методика.** Ошибки внедряются напрямую в хранилища: в S3 — изнутри сети `storage` ключами API/reconciler, в PostgreSQL — через psql. Затем запускается reconciler, сохраняется каждый JSON-отчёт. Повторяется тестом ([test_reconcile.py](student-tests/test_reconcile.py), 6 тестов) и сценарием с сохранением отчётов ([bench/reconcile_faults.py](bench/reconcile_faults.py) → [evidence/reconcile/](evidence/reconcile/)). `--grace 30 --allow-small-grace`, чтобы «старое» и «свежее» различались за секунды. + +**Результаты, сценарий A** (все виды ошибок; [faults-summary.json](evidence/reconcile/faults-summary.json)): + +| Внедрено | dry-run | `--apply` | повторный `--apply` | состояние после | +| --- | --- | --- | --- | --- | +| старый объект без строки | `orphan_object`, planned | удалён | — | объекта нет | +| `pending` старше grace с объектом / без | `stale_intent` ×2 | удалены (объект, затем строка) | — | нет ни строк, ни объекта | +| объект удалён под **используемым** `ready` (C) | `missing_object`, used | карантин `missing_object` | — | строка и ссылка из сообщения целы; content 404; прикрепить — 409 | +| объект обрезан под неиспользуемым `ready` (D) | `size_mismatch` | карантин | — | content 404 | +| тот же размер, другие байты (F, только `--verify-checksums`) | `checksum_mismatch` | карантин | — | content 404 | +| ложный карантин у здорового объекта (E) | `restored_object` | карантин снят | — | content 200 | +| свежий объект без строки, свежий `pending` (загрузка в процессе) | `fresh_*`, not_actionable | **не тронуты** | удалены, когда стали старше grace | — | +| здоровые A (используется), B | нет находок | не тронуты | — | content 200 | +| объект C возвращён «из бэкапа» | — | — | `restored_object` → карантин снят | content 200, те же байты | + +- Dry-run: отпечаток всех строк `attachments` и список ключей bucket до и после совпали (`dry_run_changed_nothing: true`). +- Повторный `--apply`: 0 действий по исправленным ключам. + +**Сценарий B — прерванный `--apply`.** 2000 старых сирот; S3 ставится на паузу во время удаления: + +- прерванный проход: exit 1, `store_unavailable`, успел удалить 440; +- после unpause осталось 1560; +- повторный запуск: exit 0, удалено 1562 (1560 этих + 2 сироты, оставшиеся от тестов); +- третий запуск: 0 запланированных действий; +- [interrupted-*.json](evidence/reconcile/). + +Ни одна `ready`-строка за весь эксперимент не удалена. + +**Тесты безопасности:** + +- `test_safety_valve_refuses_mass_deletion`: при `--max-orphan-fraction` ниже доли сирот выход 3, ничего не удалено; +- `test_store_outage_stops_the_run_without_damage[s3/postgres]`: выход 1 с JSON-отчётом; +- `test_live_uploads_are_never_hurt_by_concurrent_apply`: 40 загрузок + сообщений идут параллельно с повторяющимся `--apply --grace 8`. Все 40 вложений `ready`, без карантина, байты на месте. + +**Стоимость обхода** ([cost.json](evidence/reconcile/cost.json)) — синтетические согласованные вложения по 1 КиБ, dry-run, медиана 3 запусков, reconciler 1 CPU / 256 МБ: + +| Синтетических вложений (всего объектов в bucket) | Режим | Длительность, медиана из 3 | Запросы листинга S3 | Страницы БД | Прочитано байтов | +| --- | --- | --- | --- | --- | --- | +| 1 000 (1 285) | dry-run, страница 1000 | 0,30 с | 2 | 2 | 0 | +| 10 000 (10 285) | dry-run, страница 1000 | 1,67 с | 11 | 11 | 0 | +| 10 000 (10 285) | dry-run, страница 100 | 2,28 с (+37 %) | 103 | 103 | 0 | +| 10 000 (10 285) | dry-run, `--verify-checksums` | 48,4 с | 11 | 11 | 694 МБ (≈ 14 МБ/с) | +| 50 000 (50 285) | dry-run, страница 1000 | 8,4 с | 51 | 51 | 0 | + +Время растёт линейно: ≈ 0,17 мс на объект, ≈ 6 000 объектов/с при 1 CPU. Число запросов ровно ⌈N/страница⌉ с каждой стороны. Страница 100 даёт в 10 раз больше запросов и +37 % времени, поэтому по умолчанию страница 1000 — максимум ListObjectsV2. Проверка сумм стоит O(объёма): 694 МБ — это в основном 10-мегабайтные файлы предыдущих тестов на стенде, а не 1-килобайтная синтетика. Поэтому `--verify-checksums` — режим для редкого (например, ночного) прогона, а сверка по листингу — для частого. Память reconciler от N не зависит (лимит 256 МБ, страница хранится одна). + +**Ограничения трека:** + +- Время берётся из часов БД (строки) и хоста (LastModified объектов). Поэтому grace должен быть много больше расхождения часов; при одном хосте это выполняется. +- Без `--verify-checksums` порча с тем же размером не видна; проверка сумм читает все байты и стоит O(общего объёма). +- Reconciler не знает, «правильный» ли объект вернули из бэкапа: он проверяет размер и SHA-256 по метаданным, а значит, восстановить можно только тот же оригинал. +- Карантин — решение для оператора, не автоматическое восстановление: байты потеряны. +- Листинг S3 и чтение БД — не единый снимок. Гонки закрывают grace и перепроверка под `FOR UPDATE`, а не изоляция. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Один Docker-хост, один экземпляр SeaweedFS** без репликации (`000`): отказ диска хоста теряет и БД, и bucket. Бэкап `s3data` и `pgdata` должен быть согласованным по времени; reconciler найдёт расхождение после восстановления, но байты не вернёт. +- **SeaweedFS 4.46**: S3 gRPC-порт 18333 слушает сеть `storage` вместе с 8333 (отдельного bind для него нет); сеть internal, в ней только api, s3-init и reconciler. KEK для SSE-S3 не задан: шифрования «на диске» нет (в lab4 не требуется, вопрос lab7). +- **HTTP внутри стенда**: Basic и ключи S3 идут без TLS — допустимо для локального стенда с синтетикой, TLS/mTLS — lab7. +- **Тело загрузки в памяти** (ограниченно): потоковой передачи в S3 через multipart нет. Причина — полная проверка изображения до записи требует весь файл. Граница — слоты; lab5 перенесёт обработку в staging + worker. +- **Повторная загрузка того же файла** создаёт новое вложение (контракт это разрешает); дедупликации по SHA-256 нет. +- **Проверка JPEG** принимает файлы, которые libjpeg декодирует без единого предупреждения. Редкие реальные файлы с «extraneous bytes» от кривых кодеков получат 415 — сознательная строгость. +- **Janitor** работает в репликах API (одна за раз, advisory-замок); при большой доле мусора его стоило бы вынести в отдельный сервис, как reconciler. +- **Замеры** — один хост и по одному прогону на сценарий памяти; выводы сделаны по крупным, повторяющимся различиям, а не по процентам. diff --git a/bench/failover_series.py b/bench/failover_series.py new file mode 100644 index 00000000..b4c5dafa --- /dev/null +++ b/bench/failover_series.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and +measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL) +and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case +for detection, only the active health check can find it). + + python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes + +Timeline of one round (all times from the moment the stop command is issued): + warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy + has every replica UP (rejoin time) -> 6 s steady state. +Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed +request (+s), the first moment after which the stopped replica never answered again +(= removal from rotation), time until the restarted replica is UP in HAProxy and serves +traffic again, plus the history check (no confirmed message lost, no duplicate). +Output: /series.json (summary), /timeline--.csv (per second), +/series.md (table for REPORT.md). No tokens are written. +""" +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import platform +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "bench")) +import lbctl # noqa: E402 +from loadgen import Load # noqa: E402 + + +def docker(*args: str) -> str: + return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout + + +def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict: + load = Load(workers).start() + time.sleep(6) + t_stop = load.now() + if mode == "pause": + docker("pause", replica["id"]) + else: + docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"]) + t_stopped = load.now() + time.sleep(12) + t_start = load.now() + docker("unpause" if mode == "pause" else "start", replica["id"]) + lbctl.wait_up(replicas, timeout=90) + t_up = load.now() + # The restarted container keeps its hostname: wait until it serves requests again. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events): + break + time.sleep(0.1) + t_serves = load.now() + time.sleep(6) + load.stop() + t_end = load.now() + + failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)] + served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start] + out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0 + rows = load.per_second() + with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) + writer.writeheader() + for row in rows: + row = dict(row) + row["instances"] = row["instances"].replace(replica["hostname"], "TARGET") + writer.writerow(row) + state = json.loads(docker("inspect", replica["id"]))[0]["State"] + return { + "mode": mode, "replica": replica["name"], "instance_id": replica["hostname"], + "stop_command_seconds": round(t_stopped - t_stop, 2), + "before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end), + "failed_after_stop": len(failed_after_stop), + "failed_statuses": sorted({e[2] for e in failed_after_stop}), + "last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3), + "target_last_served_after_stop_s": round(out_of_rotation, 3), + "rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2), + "client_retries_same_key": load.retries, + "history": load.verify_history(), + "restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--output", default="evidence/failover") + parser.add_argument("--modes", default="stop,kill,pause") + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"]) + assert len(replicas) >= 2, "need at least two running replicas" + lbctl.wait_up(len(replicas)) + results = [] + for mode in args.modes.split(","): + for replica in replicas: # each replica in turn; the previous one is already restored + print(f"== {mode} {replica['name']} ...", flush=True) + result = one_round(replica, mode, len(replicas), args.workers, out) + results.append(result) + print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s", + "target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True) + meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip(), + "host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}", + "workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message " + "(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s", + "lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; " + "retry-on conn-failure + redispatch", + "api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"} + (out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8") + lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | " + "Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] + for r in results: + h = r["history"] + lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} " + f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | " + f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | " + f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | " + f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |") + (out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "", + *lines, ""]), encoding="utf-8") + print("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/loadgen.py b/bench/loadgen.py new file mode 100644 index 00000000..97a94de8 --- /dev/null +++ b/bench/loadgen.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4). + +Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page. +An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body, +exactly as a correct client must do; the history is checked afterwards for losses and +duplicates. Every request is recorded as (t, kind, status, instance, latency). +This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that). + + python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON +""" +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import threading +import time +import uuid +from collections import Counter, defaultdict + +import urllib.error +import urllib.request + +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +class Http: + """Minimal stdlib client with keep-alive per thread (http.client under urllib is not).""" + + def __init__(self, base_url: str, timeout: float): + import http.client + from urllib.parse import urlsplit + parts = urlsplit(base_url) + self.host, self.port = parts.hostname, parts.port or 80 + self.timeout = timeout + self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout) + self.conn = self.factory() + + def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None): + try: + self.conn.request(method, path, body=body, headers=headers or {}) + response = self.conn.getresponse() + data = response.read() + if response.getheader("connection", "").lower() == "close": + self.conn.close() + self.conn = self.factory() + return response.status, dict(response.getheaders()), data + except Exception: + self.conn.close() + self.conn = self.factory() + raise + + +def call(method: str, path: str, body=None, headers=None, timeout=10.0): + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json", **(headers or {})} + req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b"null") + + +def make_user(label: str) -> dict: + username = f"{label}_{uuid.uuid4().hex[:12]}" + password = "Load-password-" + uuid.uuid4().hex + status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + assert status == 201, (status, user) + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert status == 201, (status, session) + return {"id": user["id"], "token": session["token"]} + + +class Load: + def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0): + self.workers = workers + self.timeout = timeout + self.pause = pause + self.alice, self.bob = make_user("load"), make_user("load") + status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]}, + headers={"Authorization": "Bearer " + self.alice["token"]}) + assert status == 201, chat + self.chat = chat + self.stop_event = threading.Event() + self.lock = threading.Lock() + self.events: list[tuple[float, str, int, str, float]] = [] + self.confirmed: dict[str, dict] = {} + self.retries = 0 + self.t0 = time.monotonic() + self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)] + + def now(self) -> float: + return time.monotonic() - self.t0 + + def _record(self, kind: str, status: int, instance: str, latency: float) -> None: + with self.lock: + self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2))) + + def _worker(self, index: int) -> None: + http = Http(BASE_URL, self.timeout) + auth = {"Authorization": "Bearer " + self.alice["token"]} + path = f"/api/v1/chats/{self.chat['id']}/messages" + n, key = 0, uuid.uuid4().hex + text = f"w{index}-0-{key[:6]}" + while not self.stop_event.is_set(): + started = time.monotonic() + try: + status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(), + {**auth, "Idempotency-Key": key, "Content-Type": "application/json"}) + instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" + except Exception: + status, instance, data = 0, "-", b"" + self._record("POST", status, instance, time.monotonic() - started) + if status == 201: + msg = json.loads(data) + with self.lock: + self.confirmed[msg["id"]] = msg + n += 1 + key = uuid.uuid4().hex + text = f"w{index}-{n}-{key[:6]}" + elif status in (0, 502, 503, 504, 500): + with self.lock: + self.retries += 1 # same key + same text on the next iteration + started = time.monotonic() + try: + status, headers, _ = http.request("GET", path + "?limit=5", None, auth) + instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" + except Exception: + status, instance = 0, "-" + self._record("GET", status, instance, time.monotonic() - started) + if self.pause: + time.sleep(self.pause) + + def start(self) -> "Load": + self.t0 = time.monotonic() + for t in self.threads: + t.start() + return self + + def stop(self) -> None: + self.stop_event.set() + for t in self.threads: + t.join(timeout=15) + + def verify_history(self) -> dict: + """Walk the whole history with the second user: no confirmed message lost, no duplicate.""" + items, cursor = [], None + while True: + q = "?limit=100" + (f"&cursor={cursor}" if cursor else "") + status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}", + headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30) + assert status == 200, page + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + ids = [m["id"] for m in items] + texts = [m["text"] for m in items] + lost = [i for i in self.confirmed if i not in set(ids)] + changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m] + return {"history_messages": len(items), "confirmed_messages": len(self.confirmed), + "lost_confirmed": len(lost), "altered_confirmed": len(changed), + "duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)), + "unconfirmed_but_stored": len(set(ids) - set(self.confirmed))} + + # ------------------------------------------------------------------ summaries + def window(self, start: float, end: float) -> dict: + ev = [e for e in self.events if start <= e[0] < end] + ok = [e for e in ev if e[2] in (200, 201)] + lat = sorted(e[4] for e in ok) + + def pct(p: float): + return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None + span = max(end - start, 1e-9) + return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok), + "ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None, + "statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)), + "p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)} + + def per_second(self) -> list[dict]: + buckets: dict[int, list] = defaultdict(list) + for e in self.events: + buckets[int(e[0])].append(e) + rows = [] + for second in sorted(buckets): + ev = buckets[second] + rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)), + "failed": sum(1 for e in ev if e[2] not in (200, 201)), + "instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))}) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--duration", type=float, default=20) + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + load = Load(args.workers).start() + time.sleep(args.duration) + load.stop() + print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(), + "client_retries_same_key": load.retries}, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/reconcile_cost.py b/bench/reconcile_cost.py new file mode 100644 index 00000000..8de1ba60 --- /dev/null +++ b/bench/reconcile_cost.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments. + +Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory +(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)). + +Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and +SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the +reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants. +Everything synthetic is deleted at the end (objects first, then rows). + + python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import statistics +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import httpx2 as httpx + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +PASSWORD = "Bench-password-Ω-12345" +OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object +SHA = hashlib.sha256(OBJECT).hexdigest() + +PUT_SCRIPT = """ +import asyncio, sys +from messenger.config import Settings, s3_settings +from messenger.storage import Storage +DATA = bytes(range(256)) * 4 +async def main(): + s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())) + await s.open() + keys = sys.stdin.read().split() + sem = asyncio.Semaphore(32) + async def one(k): + async with sem: + await s.{op}(k{args}) + await asyncio.gather(*(one(k) for k in keys)) + await s.close() + print(len(keys)) +asyncio.run(main()) +""" + + +def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True, + encoding="utf-8", timeout=timeout, check=check) + + +def psql(sql: str) -> str: + """SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB).""" + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v", + "ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip() + + +def s3_bulk(op: str, keys: list[str]) -> None: + script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "") + compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys)) + + +def owner_and_chat() -> tuple[str, str]: + import base64 + c = httpx.Client(base_url=BASE, timeout=30) + name = "rc_" + uuid.uuid4().hex[:10] + c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status() + token = c.post("/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"] + me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json() + chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token}, + json={"title": "bench", "member_ids": [me["id"]]}).json() + return me["id"], chat["id"] + + +def add(n: int, owner: str, chat: str) -> None: + keys = ["o/" + uuid.uuid4().hex for _ in range(n)] + for i in range(0, n, 5000): + part = keys[i:i + 5000] + values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part) + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " + "object_key, status, created_at, updated_at) " + f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, " + f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', " + f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)") + s3_bulk("put", keys) + + +def reconcile(*args: str) -> dict: + out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout + return json.loads(out) + + +def measure(label: str, args: list[str], repeats: int) -> dict: + runs = [reconcile(*args) for _ in range(repeats)] + durations = [r["duration_seconds"] for r in runs] + last = runs[-1] + return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations), + "duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"], + "findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]} + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000]) + p.add_argument("--repeats", type=int, default=3) + p.add_argument("--output", default="evidence/reconcile") + args = p.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + owner, chat = owner_and_chat() + present = 0 + results = [] + steps = sorted(args.steps) + variant_step = steps[min(1, len(steps) - 1)] + try: + for target in sorted(args.steps): + started = time.perf_counter() + add(target - present, owner, chat) + present = target + print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True) + results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)}) + print(json.dumps(results[-1], ensure_ascii=False), flush=True) + if present == variant_step: # page-size and checksum variants at one size + for label, extra in (("dry-run, page 100", ["--page-size", "100"]), + ("dry-run, --verify-checksums", ["--verify-checksums"])): + results.append({"synthetic": present, **measure(label, extra, args.repeats)}) + print(json.dumps(results[-1], ensure_ascii=False), flush=True) + finally: + keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = " + f"'{chat}'").split() + if keys: + s3_bulk("delete", keys) + psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'") + print(f"removed {len(keys)} synthetic attachments", flush=True) + meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip(), + "object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)", + "note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"} + (out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/reconcile_faults.py b/bench/reconcile_faults.py new file mode 100644 index 00000000..7cb11d42 --- /dev/null +++ b/bench/reconcile_faults.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report. + +Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved): + S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an + unused ready row, same-size corruption (found only with --verify-checksums); + DB side: stale intents (with / without object), quarantine mark on a healthy row; + fresh: orphan + intent younger than --grace. + Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the + lost object from a "backup" -> apply (quarantine lifted). +Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them -> + the run stops with an error, nothing else is damaged; the next run finishes the rest; the + third run finds nothing. + + python bench/reconcile_faults.py --output evidence/reconcile +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 +from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402 + s3_put) +from test_reconcile import insert_intent, snapshot # noqa: E402 + +GRACE = ["--grace", "30", "--allow-small-grace"] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def reconcile(*args: str) -> tuple[int, dict]: + r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600) + return r.returncode, json.loads(r.stdout) + + +def save(out: Path, name: str, data) -> None: + (out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8") + + +def ours(report: dict, keys: dict[str, str]) -> dict: + """The findings of this scenario's keys only (the stand also holds other test data).""" + f = findings_by_key(report) + return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None) + for label, k in keys.items()} + + +def scenario_a(out: Path) -> dict: + api = Api() + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")} + data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content + for n, a in up.items()} + key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()} + api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]}) + keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"], + "unused, object truncated D": key["D"], "quarantined but healthy E": key["E"], + "same-size corruption F": key["F"]} + orphan = "o/" + uuid.uuid4().hex + stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex + s3_put(orphan, SAMPLE) + s3_put(stale_with, SAMPLE) + insert_intent(chat, alice, stale_with, SAMPLE, 600) + insert_intent(chat, alice, stale_without, SAMPLE, 600) + s3_delete(key["C"]) + s3_put(key["D"], data["D"][: len(data["D"]) // 2]) + s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:]) + psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'") + keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without}) + time.sleep(32) + fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex + s3_put(fresh_orphan, SAMPLE) + s3_put(fresh_intent, SAMPLE) + insert_intent(chat, alice, fresh_intent, SAMPLE, 0) + keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent}) + + before = snapshot() + _, dry = reconcile("--verify-checksums", *GRACE) + after = snapshot() + _, apply1 = reconcile("--apply", "--verify-checksums", *GRACE) + _, apply2 = reconcile("--apply", "--verify-checksums", *GRACE) + s3_put(key["C"], data["C"]) # the lost object restored from a backup + _, apply3 = reconcile("--apply", "--verify-checksums", *GRACE) + for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1), + ("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)): + save(out, name, report) + rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))} + content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content", + headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"} + present = s3_keys() + summary = { + "dry_run_changed_nothing": before == after, + "per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys), + "after restore of C": ours(apply3, keys)}, + "objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()}, + "rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()}, + "content_status_after": content, + "durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"], + apply3["duration_seconds"]], + } + print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True) + return summary + + +def scenario_b(out: Path, count: int = 2000) -> dict: + keys = ["o/" + uuid.uuid4().hex for _ in range(count)] + s3_exec("ks = sys.stdin.read().split()\n" + "sem = asyncio.Semaphore(32)\n" + "async def one(k):\n" + " async with sem:\n" + " await s.put(k, b'orphan', 'image/png', {})\n" + "await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys)) + time.sleep(32) + result: dict = {} + + def run_apply() -> None: + result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) + + t = threading.Thread(target=run_apply) + t.start() + # pause S3 as soon as deletions are under way (the reconciler is planning/listing first) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + left = len(s3_keys() & set(keys)) + if left < count: + break + time.sleep(0.2) + subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True) + t.join() + subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True) + time.sleep(5) # let S3 settle after unpause before counting + left_after_interrupt = len(s3_keys() & set(keys)) + code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) + code3, third = reconcile("--apply", *GRACE) + save(out, "interrupted-1-apply.json", result["report"]) + save(out, "interrupted-2-rerun.json", second) + save(out, "interrupted-3-third.json", third) + summary = { + "orphans_injected": count, + "interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"), + "applied": result["report"]["actions"]["applied"]}, + "orphans_left_after_interrupt": left_after_interrupt, + "rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]}, + "third": {"exit_code": code3, "planned": third["actions"]["planned"]}, + "orphans_left_at_end": len(s3_keys() & set(keys)), + } + print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True) + return summary + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--output", default="evidence/reconcile") + args = p.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + a = scenario_a(out) + b = scenario_b(out) + commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() + save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit, + "grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b}) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/rolling_under_load.py b/bench/rolling_under_load.py new file mode 100644 index 00000000..c4fd9ba9 --- /dev/null +++ b/bench/rolling_under_load.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB. + + python bench/rolling_under_load.py --output evidence/rolling-update + +Writes /summary.json (errors, RPS, history check, revisions before/after), +/timeline.csv (per second ok/failed/instances) and /script-output.txt. +""" +from __future__ import annotations + +import argparse +import csv +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "bench")) +import lbctl # noqa: E402 +from loadgen import Load # noqa: E402 + + +def revisions() -> dict[str, str]: + result = {} + for r in lbctl.replicas(): + if r["state"] == "running": + label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}', + r["id"]], capture_output=True, text=True).stdout.strip() + result[r["name"]] = f"{r['hostname']} revision={label}" + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--output", default="evidence/rolling-update") + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + before = revisions() + load = Load(args.workers).start() + time.sleep(5) + t_begin = load.now() + bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe") + if os.name == "nt" else "sh") + env = {**os.environ, "PYTHON": sys.executable} + script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env, + capture_output=True, text=True, encoding="utf-8", timeout=900) + t_end = load.now() + time.sleep(5) + load.stop() + (out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n" + + script.stdout + script.stderr, encoding="utf-8") + with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) + writer.writeheader() + writer.writerows(load.per_second()) + summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1), + "before": before, "after": revisions(), + "during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()), + "client_retries_same_key": load.retries, "history": load.verify_history(), + "workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"} + (out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")}, + ensure_ascii=False, indent=2)) + print(json.dumps(summary["during_update"], ensure_ascii=False)) + return 0 if script.returncode == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/upload_memory.py b/bench/upload_memory.py new file mode 100644 index 00000000..e2c6f352 --- /dev/null +++ b/bench/upload_memory.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads. + +Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies +waiting for a slot stay in TCP buffers, not in the process) + one image check at a time +(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once. + +Method (repeatable, the stand must be up): +* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup + counter memory.peak starts from an idle process (it never decreases during a container's life); +* idle memory.current of each replica is read; then N clients upload at the same moment through + the LB (a barrier releases them together), each a different file; +* during the run memory.current of every replica is sampled every 50 ms inside the container; + afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of + PID 1 (the Python server alone) are read; +* the normal configuration is restored at the end. + + python bench/upload_memory.py --output evidence/memory +""" +from __future__ import annotations + +import argparse +import io +import json +import os +import platform +import statistics +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx2 as httpx +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +PASSWORD = "Bench-password-Ω-12345" +MiB = 1024 * 1024 + + +def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str: + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout, + env={**os.environ, **(env or {})}).stdout + + +def replicas() -> list[str]: + return sh("docker", "compose", "ps", "-q", "api").split() + + +def cg(cid: str, name: str) -> int: + return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0]) + + +def memory_stat(cid: str, key: str) -> int: + for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines(): + name, value = line.split() + if name == key: + return int(value) + return 0 + + +def vmhwm(cid: str) -> int: + for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines(): + if line.startswith("VmHWM:"): + return int(line.split()[1]) * 1024 + return 0 + + +def container_events() -> dict[str, str]: + """restart count + OOM-kill events per container (cgroup memory.events oom_kill counter).""" + out = {} + for cid in sh("docker", "compose", "ps", "-q").split(): + name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split() + events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines()) + out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}" + return out + + +def recreate(env: dict) -> None: + sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300) + sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP + time.sleep(3) + + +class Sampler(threading.Thread): + """memory.current of one container every 50 ms, read by a shell loop inside it.""" + + def __init__(self, cid: str): + super().__init__(daemon=True) + self.cid = cid + self.values: list[int] = [] + self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c", + "while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"], + stdout=subprocess.PIPE, text=True) + + def run(self) -> None: + for line in self.proc.stdout: + if line.strip().isdigit(): + self.values.append(int(line)) + + def stop(self) -> None: + self.proc.kill() + + +def actor(client: httpx.Client) -> tuple[str, str]: + name = "mem_" + uuid.uuid4().hex[:10] + client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status() + import base64 + basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode() + token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"] + me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json() + chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token}, + json={"title": "mem", "member_ids": [me["id"]]}).json() + return token, chat["id"] + + +def big_jpeg(width: int, height: int, seed: int) -> bytes: + """Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check.""" + image = Image.linear_gradient("L").resize((width, height)).convert("RGB") + image.putpixel((seed % width, 0), (seed % 255, 0, 0)) + out = io.BytesIO() + image.save(out, "JPEG", quality=90, progressive=True, subsampling=0) + return out.getvalue() + + +def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict: + recreate(env) + ids = replicas() + idle = {cid: cg(cid, "memory.current") for cid in ids} + client = httpx.Client(base_url=BASE, timeout=120) + token, chat_id = actor(client) + barrier = threading.Barrier(len(files)) + results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item] + + def upload(i: int, data: bytes, ctype: str) -> None: + c = httpx.Client(base_url=BASE, timeout=120) + barrier.wait() + started = time.perf_counter() + r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data, + headers={"Authorization": "Bearer " + token, "Content-Type": ctype}) + reason = r.json()["error"]["message"] if r.status_code != 201 else "" + results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"), reason) + c.close() + + s3_id = sh("docker", "compose", "ps", "-q", "s3").strip() + restarts_before = container_events() + samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)] + for s in samplers: + s.start() + time.sleep(0.5) + threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)] + wall = time.perf_counter() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.perf_counter() - wall + time.sleep(0.5) + for s in samplers: + s.stop() + per_replica = {} + for cid, s in zip(ids, samplers[:-1]): + hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip() + per_replica[hostname] = { + "idle_mib": round(idle[cid] / MiB, 1), + "peak_mib": round(cg(cid, "memory.peak") / MiB, 1), + "sampled_max_mib": round(max(s.values or [0]) / MiB, 1), + "python_vmhwm_mib": round(vmhwm(cid) / MiB, 1), + "uploads_served": sum(1 for r in results if r[2] == hostname), + "limit_mib": round(cg(cid, "memory.max") / MiB, 1), + } + per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"] + - per_replica[hostname]["idle_mib"], 1) + lat = sorted(r[1] for r in results if r[0] == 201) + codes: dict[str, int] = {} + reasons: dict[str, int] = {} + for r in results: + codes[str(r[0])] = codes.get(str(r[0]), 0) + 1 + if r[3]: + reasons[r[3]] = reasons.get(r[3], 0) + 1 + summary = { + "scenario": name, "settings": env, "clients": len(files), + "s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1), + "s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1), + "s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1), + # OOM kills / restarts DURING the scenario (any container of the stand): must be empty + "incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)}, + "file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2), + "types": sorted({t for _, t in files}), "status_codes": codes, "error_reasons": reasons, "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False), flush=True) + return summary + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--clients", type=int, default=16) + p.add_argument("--output", default="evidence/memory") + args = p.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + print("generating files ...", flush=True) + pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)] + jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)] + base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"} + n, n2 = args.clients, 2 * args.clients + scenarios = [ + (f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]), + (f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs), + (f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs), + (f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs), + (f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs), + ] + results = [] + sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process + sh("docker", "compose", "up", "-d", "--wait", "s3") + try: + for name, env, files in scenarios: + results.append(run_scenario(name, env, files)) + finally: + recreate({}) # back to the compose defaults + meta = { + "date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(), + "commit": sh("git", "rev-parse", "--short", "HEAD").strip(), + "api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", + replicas()[0]).strip(), + "jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2), + } + (out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1), + encoding="utf-8") + print(json.dumps(meta, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..6899f048 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,338 @@ +# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) +# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes). +# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md): +# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored) +# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...) +# docker compose --env-file .env up -d --build --wait +# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb. +# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and +# every service gets only the secrets it needs. Only the LB publishes ports (loopback only). +# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and +# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside). +name: mtusi-lab4 + +x-api-env: &api-env + APP_ENV: ${APP_ENV:-lab} + LOG_LEVEL: ${LOG_LEVEL:-info} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: ${SESSION_STORE:-valkey} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + # Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests. + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8} + DOCS_ENABLED: ${DOCS_ENABLED:-true} + MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function} + # Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits. + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4} + UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5} + UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30} + IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10} + IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1} + JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256} + PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900} + JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60} + # Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures. + FAULT_POINTS: ${FAULT_POINTS:-} + +services: + postgres: + image: postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + # pg_stat_statements: per-statement counters for the experiments. + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -q"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery + restart: unless-stopped + networks: [data] + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"} + + valkey: + image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 + # The password goes into a config include readable only by the valkey user, + # so it never appears in the process arguments. docker-entrypoint.sh drops root. + command: + - sh + - -c + - >- + umask 077 && + printf 'requirepass %s\n' "$$(cat /run/secrets/valkey_password)" > /tmp/requirepass.conf && + chown valkey /tmp/requirepass.conf && + exec docker-entrypoint.sh valkey-server /etc/valkey/valkey.conf + secrets: [valkey_password] + volumes: + - ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro + - valkeydata:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + restart: unless-stopped + networks: [data] + deploy: + resources: + limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} + + # Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one + # process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities + # and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on + # 127.0.0.1 inside the container only. + s3: + image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 + entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"] + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + # Go heap target below the container limit: the GC returns memory before the cgroup OOM. + GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB} + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_api_access_key_id + - s3_api_secret_access_key + - s3_reconciler_access_key_id + - s3_reconciler_secret_access_key + volumes: + - ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro + - s3data:/data + healthcheck: + # Unauthenticated health endpoint of the S3 gateway (200 once it serves requests). + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + stop_grace_period: 30s + restart: unless-stopped + networks: [storage] + deploy: + resources: + limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"} + + # One-shot schema migration: runs ONCE per `up`, before any API replica starts + # (api depends_on service_completed_successfully). Replicas never migrate themselves. + migrate: + build: + context: . + args: + BUILD_REVISION: ${BUILD_REVISION:-dev} + image: ${API_IMAGE:-mtusi-messenger:lab4} + command: ["python", "-m", "messenger.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + restart: "no" + networks: [data] + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} + + # One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model + # (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3. + s3-init: + image: ${API_IMAGE:-mtusi-messenger:lab4} + command: ["python", "-m", "messenger.s3_bootstrap"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id + S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_api_access_key_id + - s3_api_secret_access_key + - s3_reconciler_access_key_id + - s3_reconciler_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully # same image: built once by `migrate` + healthcheck: + disable: true + restart: "no" + networks: [storage] + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} + + api: + build: + context: . + args: + BUILD_REVISION: ${BUILD_REVISION:-dev} + image: ${API_IMAGE:-mtusi-messenger:lab4} + environment: *api-env + secrets: + - database_url + - cache_url + - source: s3_api_access_key_id + target: s3_access_key_id + - source: s3_api_secret_access_key + target: s3_secret_access_key + # No `ports` and no container_name: replicas are reachable only through `lb`, + # and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname. + expose: ["8080"] + healthcheck: + test: ["CMD", "python", "-m", "messenger.healthcheck"] + interval: 2s + timeout: 2s + retries: 3 + start_period: 5s + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s) + restart: unless-stopped + networks: [edge, data, storage] + deploy: + replicas: ${API_REPLICAS:-2} + resources: + # 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime; + # measured in evidence/memory/ (bench/upload_memory.py). + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"} + # No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound + # of the experiments is a real bound, and running out of it is visible, not hidden in swap. + memswap_limit: ${API_MEMORY:-512m} + + lb: + image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + # The ONLY public entry point, on the loopback interfaces only. Both IPv4 and IPv6: + # on Windows `localhost` resolves to ::1 first, and a refused ::1 connection costs ~2 s + # per request before the fallback to 127.0.0.1 (measured in lab 3: 2.0 s vs 0.007 s). + - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" + - "[::1]:${APP_PUBLISH_PORT:-8080}:8080" + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas) + healthcheck: + # Config is valid and the process answers; whether any API is ready is the LB's own job. + test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"] + interval: 5s + timeout: 3s + retries: 3 + depends_on: + api: + condition: service_healthy + restart: unless-stopped + networks: [edge] + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} + + # Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on + # demand, dry-run by default (read-only on both stores): + # docker compose run --rm reconciler # report only + # docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine + reconciler: + profiles: ["tools"] + image: ${API_IMAGE:-mtusi-messenger:lab4} + entrypoint: ["python", "-m", "messenger.reconcile"] + command: [] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} + secrets: + - database_url + - source: s3_reconciler_access_key_id + target: s3_access_key_id + - source: s3_reconciler_secret_access_key + target: s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + restart: "no" + networks: [data, storage] + deploy: + resources: + limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m} + +networks: + edge: + data: + internal: true # no route outside: stores talk only to api/migrate + storage: + internal: true # S3 API reachable only by api, s3-init, reconciler + +volumes: + pgdata: + valkeydata: + s3data: + +secrets: + postgres_password: + file: ./secrets/postgres_password + valkey_password: + file: ./secrets/valkey_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_api_access_key_id: + file: ./secrets/s3_api_access_key_id + s3_api_secret_access_key: + file: ./secrets/s3_api_secret_access_key + s3_reconciler_access_key_id: + file: ./secrets/s3_reconciler_access_key_id + s3_reconciler_secret_access_key: + file: ./secrets/s3_reconciler_secret_access_key diff --git a/docs/CONSISTENCY.md b/docs/CONSISTENCY.md new file mode 100644 index 00000000..4614246d --- /dev/null +++ b/docs/CONSISTENCY.md @@ -0,0 +1,73 @@ +# Согласованность PostgreSQL и S3 (lab4: уровень 4 и трек 2) + +Байты вложения лежат в приватном bucket S3 (SeaweedFS), метаданные — в таблице `attachments` PostgreSQL. Общей транзакции у двух хранилищ нет, поэтому порядок записи и механизм уборки определяют, что остаётся после любого сбоя. + +## Инвариант и порядок записи + +**Объект никогда не существует без строки, которая называет его ключ.** Строка появляется раньше объекта, а при уборке исчезает позже. + +```text +upload: [проверки, тело в памяти, полная проверка изображения] + 1. INSERT attachments(status='pending', object_key=o/<128 бит>) COMMIT ← намерение + 2. S3 PUT o/ (Content-MD5) + 3. UPDATE status='ready' WHERE status='pending' COMMIT + 4. 201 ready +janitor: SELECT … WHERE status='pending' AND старше PENDING_GRACE FOR UPDATE SKIP LOCKED + S3 DELETE (идемпотентно) → DELETE строки COMMIT +``` + +| Сбой | Что остаётся | Кто и когда убирает | Что видит клиент | +| --- | --- | --- | --- | +| До шага 1 (лимит, 415, обрыв тела, таймаут) | ничего: до полной проверки в хранилища ничего не пишется | — | 4xx/обрыв | +| Шаг 1 не прошёл (БД недоступна) | ничего | — | 503 | +| Между 1 и 2, S3 PUT упал или неизвестно, записан ли объект | строка `pending` (+ возможно объект) | janitor через `PENDING_GRACE_SECONDS` (900 с): удаляет объект, затем строку | 503, повтор создаёт новое вложение (контракт это разрешает) | +| Между 2 и 3 (процесс упал, БД пропала) — `FAULT_POINTS=after_s3_put` | строка `pending` + объект | janitor | 503 | +| Шаг 3 закоммичен, ответ потерян — `FAULT_POINTS=after_ready_commit` | корректное `ready`-вложение | **никто не трогает**: janitor работает только с `pending` | 503; файл скачивается по id ([test_consistency.py](../student-tests/test_consistency.py)) | + +В обработчике запроса **компенсации нет**. Если ответ на COMMIT потерян, неизвестно, стала ли строка `ready`. Удалить объект «на всякий случай» значило бы получить `ready` без байтов — именно это преподаватель нашёл в чужом PR. Решение принимает только janitor, по зафиксированному состоянию и под блокировкой строки: «зависшая» загрузка, если бы она ещё шла, ждала бы на этой строке и затем получила бы 503. + +**Почему grace 900 с безопасен.** Самая долгая загрузка: ожидание слота 5 с + тело 30 с + проверка 10 с + S3 PUT ≈ 4 с, итого < 60 с. Grace на порядок больше. Reconciler отказывается работать с `--grace` < 60 с без явного флага `--allow-small-grace` (флаг нужен только тестам). + +**Когда честно вернуть 201.** Только после шага 3: объект записан (S3 подтвердил PUT и проверил Content-MD5), строка `ready` закоммичена. До этого клиент не получает id. + +## Reconciler (трек 2) + +`python -m messenger.reconcile` — отдельный CLI в том же образе, compose-сервис `reconciler` (профиль `tools`, запускается по требованию). У него свои ключи S3 (`Read+Write+List` на bucket), у API права `List` нет. + +### Как он обходит хранилища + +Оба хранилища читаются как упорядоченные по ключу потоки, и над ними выполняется merge-join: + +- S3: `ListObjectsV2(prefix=o/, StartAfter, MaxKeys=1000)` — ключи в порядке байтов; +- PostgreSQL: keyset-страницы `SELECT … FROM attachments WHERE object_key > $1 ORDER BY object_key LIMIT 1000` — колонка `COLLATE "C"`, тот же порядок байтов, уникальный индекс. + +Память не зависит от размера bucket. Стоимость прохода ≈ N/1000 запросов листинга + N/1000 страниц БД; с `--verify-checksums` добавляется чтение всех байтов. + +### Что находит и что исправляет `--apply` + +| Вид | Условие | `--apply` | +| --- | --- | --- | +| `orphan_object` | объект без строки, старше `--grace` | удалить объект (перед этим заново проверить, что строки нет и объект всё ещё старый) | +| `stale_intent` | строка `pending` старше `--grace` (с объектом или без) | под `FOR UPDATE`: удалить объект, затем строку | +| `missing_object` | строка `ready`, объекта нет, старше `--grace` | **карантин** строки: `lost_at`, `lost_reason` | +| `size_mismatch` | у `ready` размер объекта ≠ `size_bytes` | карантин | +| `checksum_mismatch` | SHA-256 объекта ≠ сохранённому (`--verify-checksums`) | карантин | +| `restored_object` | строка в карантине, объект снова на месте и верный | снять карантин | +| `fresh_*` | то же, но моложе `--grace` (загрузка в процессе) | ничего | + +Карантин: `GET …/content` → 404 «оригинал утрачен», прикрепить к новому сообщению нельзя (409, как failed-вложение). Метаданные и ссылки из старых сообщений сохраняются. Карантин обратим: когда объект восстановят из резервной копии, следующий проход его снимет. + +### Почему исправление безопасно + +1. **Dry-run read-only конструктивно, а не «по if-у».** Сессия PostgreSQL открывается с `default_transaction_read_only=on`: любая запись упадёт в самой БД. Объект S3 в dry-run обёрнут в `ReadOnlyStorage` без метода `delete`. Тест сравнивает отпечаток всех строк `attachments` и список ключей bucket до и после dry-run. Это закрывает ошибку, которую нашли в чужом PR: «dry-run продолжает частично завершённое удаление». +2. **`ready`-строки никогда не удаляются**, ни используемые, ни неиспользуемые: только обратимый карантин. Удаляются лишь объекты без строк и строки-намерения `pending`. Ссылки `message_attachments → attachments` защищены FK, поэтому используемое вложение не удалить даже по ошибке. +3. **Всё, что моложе `--grace`, не трогается.** +4. **Перепроверка перед каждым действием**, под `FOR UPDATE`, где есть строка. `post_message_v2` берёт `FOR SHARE` на прикрепляемые вложения, поэтому карантин и отправка сообщения сериализуются. +5. **Объект первым, строка второй.** Прерванный проход оставляет состояние, которое следующий проход доводит до конца. Повторный запуск идемпотентен. +6. **Предохранитель.** Если «сиротами» выглядят больше `--max-orphan-fraction` (50 %) объектов — например, reconciler направили не в ту БД, — `--apply` завершается с кодом 3, ничего не удалив. +7. **Отказ хранилища** во время прохода: выход с кодом 1 и JSON-отчётом `store_unavailable`; уже сделанное зафиксировано, остальное сделает следующий запуск. + +### Janitor и reconciler — зачем оба + +- **Janitor** (в каждой реплике API, раз в ~60 с, advisory-замок — работает одна реплика за раз) дёшев: частичный индекс по `pending` и ничего больше. Он убирает типичный мусор — незавершённые загрузки. +- **Reconciler** проходит оба хранилища целиком и находит то, о чём БД не знает: объекты без строк (ручные действия, баги прошлых версий, восстановление bucket из другой копии), пропавшие и испорченные объекты. Его запускают по расписанию или вручную, всегда сначала в dry-run. diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md new file mode 100644 index 00000000..815b0dcb --- /dev/null +++ b/docs/DATA_MODEL.md @@ -0,0 +1,47 @@ +# Модель данных lab4 + +Схема: [0001_init.sql](../migrations/0001_init.sql) (lab2), [0002_idempotency_keys.sql](../migrations/0002_idempotency_keys.sql) и [0003_post_message_function.sql](../migrations/0003_post_message_function.sql) (lab3), [0004_attachments.sql](../migrations/0004_attachments.sql) (lab4). Миграции применяет один раз отдельный сервис `migrate` до старта реплик API; миграции lab3 и lab4 только добавляют объекты (0004 ещё ослабляет `CHECK` длины текста), поэтому совместимы с работающей предыдущей версией. Байты вложений живут **только** в приватном bucket S3 (SeaweedFS), в PostgreSQL — их метаданные; согласованность двух хранилищ — [CONSISTENCY.md](CONSISTENCY.md). Источник истины для пользователей, чатов, участников и сообщений — PostgreSQL 18. Для сессий источник истины — хранилище, выбранное через `SESSION_STORE`. По умолчанию это Valkey с AOF, его можно переключить на таблицу `sessions` в PostgreSQL. + +## Таблицы и ограничения + +| Таблица | Ключ | Ограничения | Зачем | +| --- | --- | --- | --- | +| `users` | `id uuid` | `UNIQUE (username)`, `CHECK` формата username, длина `display_name` | уникальность имени держит БД, а не код: гонку регистрации разрешает уникальный индекс | +| `chats` | `id uuid` | FK `created_by → users`, длина `title` | `last_message_at` — «часы» чата для упорядочивания сообщений | +| `chat_members` | `(chat_id, user_id)` | FK на `chats` и `users`, `UNIQUE (chat_id, position)` | `position` хранит порядок `member_ids`; `chat_created_at` — копия времени чата для keyset-списка чатов | +| `messages` | `id uuid` | FK `(chat_id, sender_id) → chat_members`, длина `text` | отправитель обязан быть участником — это гарантирует сама БД | +| `sessions` | `token_hash bytea` | FK `user_id → users`, длина хеша 32 байта | хранится SHA-256 токена; сам токен не хранится нигде | +| `idempotency_keys` | `(user_id, chat_id, key)` | FK на `users`, `chats`; `message_id → messages` (DEFERRABLE, `UNIQUE`); `CHECK` формата ключа и длины SHA-256 | общая для всех реплик идемпотентность отправки: ключ → отпечаток тела → сообщение. Хранится ≥ `IDEMPOTENCY_TTL_SECONDS` (24 ч), очистка — пачками раз в 10 мин под advisory-замком | +| `attachments` (lab4) | `id uuid` | FK `(chat_id, owner_id) → chat_members`; `UNIQUE object_key` (`COLLATE "C"`, формат `o/<32 hex>`); `CHECK` статуса `pending/ready`, типа `image/png|jpeg`, размера 1…10 MiB, пикселей ≤ 40 млн, длины SHA-256, карантина (`lost_at`/`lost_reason` только у `ready`) | метаданные оригинала; `object_key` генерирует сервер (128 случайных бит), с id и filename не связан. `pending` — намерение загрузки (строка раньше объекта), `ready` — объект записан. Частичный индекс `(created_at) WHERE status='pending'` — для janitor | +| `message_attachments` (lab4) | `(message_id, position)` | FK на `messages` и `attachments`, `UNIQUE (message_id, attachment_id)`, индекс по `attachment_id` | целостность ссылок (вложение, на которое ссылается сообщение, удалить нельзя) и обратный поиск «используется ли вложение» | +| `schema_migrations` | `version` | checksum файла | какие миграции применены; изменённую задним числом миграцию мигратор не применит | + +**Почему `bytea` для текста.** Контракт разрешает символ U+0000, а тип `text` в PostgreSQL не может содержать байт NUL (`DataError`). Поэтому `display_name`, `title` и `text` хранятся как байты UTF-8. Длину в code points проверяет API, а `CHECK` в байтах — страховка (1 code point ≤ 4 байта). + +## Индексы и запросы + +- `messages_chat_keyset (chat_id, created_at, id)`. Первая страница — `WHERE chat_id=$1 ORDER BY created_at, id LIMIT n`, следующая — `WHERE chat_id=$1 AND (created_at, id) > ($2, $3) ORDER BY ... LIMIT n`. Сравнение строк `(created_at, id) > (...)` попадает в `Index Cond`: поиск начинается прямо с позиции cursor и читает только `n+1` строк. Запросов два, а не один с `($2 IS NULL OR ...)`: иначе при generic plan условие cursor уходит в `Filter`, и поздняя страница сканирует всю историю чата до неё. +- `chat_members_user_keyset (user_id, chat_created_at, chat_id)` — список чатов пользователя тем же keyset-способом. +- `users_username_key` — вход по username и уникальность. +- `sessions_expires_at` — очистка истёкших сессий. + +## Транзакционные границы + +| Операция | Транзакция | Что гарантирует | +| --- | --- | --- | +| Регистрация | один `INSERT` | `UniqueViolation` → 409; ровно одна строка при гонке | +| Создание чата | `INSERT chats` → `INSERT chat_members` (все участники одним запросом) → `COMMIT` | неизвестный участник вызывает FK-ошибку **после** вставки чата; откат удаляет и чат, и частичное членство | +| Сообщение (lab3, с ключом) | проверка членства → `INSERT idempotency_keys … ON CONFLICT DO UPDATE … WHERE ключ старше 24 ч` → если вставлен: `UPDATE chats … RETURNING` → `INSERT messages`; если нет: прочитать сохранённое сообщение и сравнить отпечаток → `COMMIT`. По умолчанию всё это — одна SQL-функция `post_message` в autocommit (одно обращение к БД) | ключ, сообщение и «результат» коммитятся вместе или не коммитятся вовсе: «висящего» ключа нет. Конкурентный повтор ждёт на уникальном индексе завершения первой транзакции и затем видит её результат (READ COMMITTED, новый снимок на каждый оператор). Другое тело → 409, тот же — тот же Message | +| Сообщение с `attachment_ids` (lab4) | в той же функции `post_message_v2`: проверка членства → проверка вложений (`ready`, этого чата, не в карантине; `FOR SHARE`) → только если всё хорошо — захват ключа → `UPDATE chats` → `INSERT messages (…, attachment_ids)` → `INSERT message_attachments` | чужое/неизвестное/незавершённое вложение — 404, в карантине — 409, ключ при этом не «сгорает». Порядок `attachment_ids` хранится массивом в строке сообщения: страница истории остаётся одним диапазонным сканом индекса | +| Загрузка вложения (lab4) | `INSERT attachments … 'pending'` (autocommit) → S3 PUT (Content-MD5) → `UPDATE … SET status='ready' WHERE status='pending'` (autocommit) → 201 | 201 только когда и объект, и строка `ready` зафиксированы. Сбой между шагами оставляет `pending`, его убирает janitor после `PENDING_GRACE_SECONDS`. Компенсации в обработчике запроса нет: после потерянного ответа COMMIT неизвестно, `ready` ли строка | +| Сообщение (lab2-часть) | проверка членства → `UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + 1 мкс) RETURNING` → `INSERT messages` → `COMMIT` | строка чата заблокирована до конца транзакции, поэтому писатели одного чата идут по очереди. Время сообщений строго растёт **в порядке коммитов**: страница по cursor не пропустит сообщение, закоммиченное «задним числом» | +| Вход | `SELECT` пользователя, проверка Argon2id, запись сессии | неверный пароль ничего не пишет | +| Logout | `DELETE` сессии (или `DEL` в Valkey) | повтор с тем же токеном — 401 | + +Чтение истории не требует snapshot между HTTP-запросами (так в контракте): новое сообщение появится на следующей странице. Список чатов сортируется по `chats.created_at = clock_timestamp()` без блокировки. Два чата, создаваемые одновременно, могут закоммититься не в порядке времени. Тогда у обхода, который уже начался, такой чат может оказаться «до» cursor; новый обход его увидит. Чаты создаются редко, поэтому мы не сериализуем их глобальным замком; это ограничение описано в REPORT.md. + +## Сессии + +- Токен: `secrets.token_urlsafe(32)` — 256 бит случайности, в ответе 43 символа. В хранилище лежит только SHA-256 токена. Для случайного 256-битного значения быстрого хеша достаточно, медленный KDF нужен только для низкоэнтропийных паролей. +- `SESSION_STORE=valkey` (по умолчанию): ключ `session:`, значение `{user_id, expires_at}`. Срок жизни задаётся абсолютным `PXAT expires_at`, поэтому рестарт его не продлевает. Включён `appendonly yes` + `appendfsync always`: подтверждённые вход и logout сохраняются до ответа клиенту. Valkey — **единственное** хранилище сессий, это не кэш над PostgreSQL. Поэтому нет второй копии, которая могла бы «оживить» отозванный токен после отказа. Если Valkey недоступен, проверить сессию нельзя: защищённые запросы и logout получают 503, анонимный доступ не открывается. +- `SESSION_STORE=postgres`: таблица `sessions`, проверка `expires_at > now()` в запросе. diff --git a/docs/DEFENSE.md b/docs/DEFENSE.md new file mode 100644 index 00000000..f1381564 --- /dev/null +++ b/docs/DEFENSE.md @@ -0,0 +1,39 @@ +# Ответы на вопросы защиты ЛР 4 + +## Почему файл в Docker volume одной API-реплики не равен объектному хранилищу? + +- **Volume принадлежит одному контейнеру.** Файл, загруженный через реплику api-1, лежит только у неё. Запрос, который LB отправил на api-2, его не найдёт. Если api-1 остановлена (failover, rolling update, scale down), файл недоступен вовсе. В нашем стенде у сервиса `api` нет ни одного volume: реплики взаимозаменяемы, и тест «загрузка → kill обслужившей реплики → скачивание через другую» проходит ([test_storage_failures.py](../student-tests/test_storage_failures.py)). +- **Общий bind-mount на одном хосте** тоже не выход. Он привязывает все реплики к одной машине, не даёт атомарной записи объекта, проверки целостности (Content-MD5), отдельных прав доступа и HTTP-API для других сервисов (worker в lab5 возьмёт тот же объект по ключу). +- **S3-сервер** — отдельный процесс со своим жизненным циклом и своим volume (`s3data`): он переживает перезапуск и замену API. Права в нём задаются по ключам: у API — только Read+Write своего bucket, без List и без создания bucket. + +## Когда можно честно вернуть 201? + +Когда оба хранилища подтвердили запись: S3 ответил на PUT, проверив Content-MD5, и после этого закоммичен `UPDATE attachments SET status='ready'`. До этого клиент не получает id вложения. Перед записью проходят все проверки: лимиты, полная проверка изображения, SHA-256. Ответ 201 на «принял в память» или «записал строку, а PUT ещё идёт» был бы ложью: после сбоя по такому id ничего не скачать. Подробно: [CONSISTENCY.md](CONSISTENCY.md). + +## Что произойдёт, если S3 PUT прошёл, а COMMIT в PostgreSQL — нет? + +Строка-намерение `pending` с ключом объекта закоммичена **до** PUT, поэтому объект не «потеряется»: БД знает его ключ. + +- Клиент получает 503 и может повторить: получится новое вложение, контракт это разрешает. +- Строка остаётся `pending`, вложение невидимо (GET → 404, прикрепить → 404). +- Janitor через `PENDING_GRACE_SECONDS` (900 с) под `FOR UPDATE` удаляет объект, затем строку. Проверено внедрением отказа `FAULT_POINTS=after_s3_put`: строка и объект исчезают ([test_consistency.py](../student-tests/test_consistency.py)). +- Если упал не COMMIT, а **ответ** на него (COMMIT прошёл), в обработчике ничего не удаляется: неизвестно, стала ли строка `ready`. Проверено `FAULT_POINTS=after_ready_commit`: файл цел и скачивается. +- Объекты, о которых БД не знает вовсе (ручные действия, старые версии), находит reconciler (трек 2) и удаляет, если они старше grace. + +## Почему случайный object key и приватный bucket не заменяют ACL приложения? + +- **Приватный bucket** запрещает анонимный доступ к S3, но решение «может ли *этот пользователь* читать *этот файл*» принимает только приложение: оно знает участников чата. Ключи S3 у API одни на всех пользователей. Поэтому каждый `GET /attachments/{id}` и `/content` проверяет членство в чате, посторонний получает 404 (не 403 — существование чужого вложения не раскрывается). +- **Случайный ключ** — защита от перебора, а не авторизация. Ключ может утечь (логи, отладка, бывший участник чата). Удалённого из чата пользователя в нашем API нет, но в общем случае доступ должен отзываться — а ключ, если бы он был единственной защитой, остался бы действующим навсегда. Поэтому ключ не попадает ни в один ответ API, а байты отдаются только через авторизованный API, без redirect на S3. +- То же проверено снаружи: анонимный GET/LIST к S3 → 403 (bootstrap `s3-init` и тест), а запрос с ключом API работает — значит, права проверяет S3, а доступ пользователей — приложение. + +## Частые уточняющие вопросы + +**Как ограничена память?** Одна загрузка — один буфер ≤ 10 MiB (при известном Content-Length выделяется ровно этот объём, без копий). На реплику одновременно принимается не больше `UPLOAD_CONCURRENCY=4` тел. Остальные ждут ≤ 5 с, их байты остаются в TCP-буферах (backpressure), потом 503 + Retry-After. Полная проверка изображения выполняется по одной (`IMAGE_VERIFY_CONCURRENCY=1`). Пиксели проверяются по заголовку ещё до чтения остального тела. Замер ≥ 10 одновременных загрузок: [REPORT.md](../REPORT.md#эксперименты-и-отказы). + +**Почему не Pillow для проверки?** Для PNG Pillow создаёт растр целиком (40 Мпикс RGBA = 160 МБ), а наш валидатор проверяет CRC и zlib-поток потоково и ничего не хранит. Для JPEG Pillow игнорирует предупреждения libjpeg и возвращает серую картинку для файла без данных после SOS. `djpeg -strict` считает любое предупреждение ошибкой, а `-scale 1/8` декодирует все блоки, но пишет растр в 64 раза меньше. + +**Что если S3 недоступен?** `/health/ready` → 503 (S3 входит в обязательный синхронный путь). Загрузка и скачивание → 503 за ≤ `S3_TIMEOUT_SECONDS` (3 с). HAProxy переключается на backend `api_core` с проверкой `/health/ready/core` (БД + Valkey): чаты и текстовые сообщения продолжают работать. Если S3 потеряла одна реплика, она DOWN в основном backend, и трафик идёт на другую. + +**Почему SeaweedFS, а не MinIO?** Образы MinIO в Docker Hub больше не публикуются (репозиторий `minio/minio` удалён, проверено при выборе). SeaweedFS развивается и умеет identity с правами на bucket (`Read:bucket`, `Write:bucket`, `List:bucket`). Закреплён по digest (4.46, сентябрь 2026). Его web-интерфейсы (master, filer, volume) слушают только 127.0.0.1 внутри контейнера, телеметрия выключена. + +**Где dry-run гарантирует, что ничего не меняется?** Не в коде планировщика, а в окружении: сессия PostgreSQL в dry-run открыта с `default_transaction_read_only=on`, объект S3 обёрнут без метода `delete`. Тест сравнивает отпечаток всей таблицы `attachments` и список ключей bucket до и после. diff --git a/docs/PR.md b/docs/PR.md new file mode 100644 index 00000000..3b5b2d7e --- /dev/null +++ b/docs/PR.md @@ -0,0 +1,20 @@ +## Лабораторная 4 + +Автор / группа: Боданюк Алексей, БВТ2302 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: трек 2 «Согласованность двух хранилищ» (reconciler S3 ↔ PostgreSQL) + +Что реализовано и что перенесено: реализация lab3 перенесена отдельным коммитом без изменений. Добавлено: S3-хранилище SeaweedFS 4.46 (приватный bucket, идемпотентный bootstrap с проверкой прав, отдельные ключи для API / reconciler / admin), вложения PNG/JPEG (метаданные в PostgreSQL, байты в S3; полная проверка целостности до записи; лимиты до чтения и декодирования), `attachment_ids` в сообщениях, скачивание через API без redirect, деградированный режим LB при отказе S3, janitor незавершённых загрузок, reconciler с dry-run, внедрение отказов, замеры памяти и стоимости обхода. Подробно — REPORT.md. +Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`python3 scripts/init_secrets.py`, `cp .env.example .env`, `docker compose --env-file .env up -d --build --wait`). +Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев», вывод в evidence/. +Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги». +Источники и использование ИИ: REPORT.md, шапка (код, тесты и отчёт написаны с помощью Claude Code; все замеры реально выполнены). + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. + +🤖 Generated with [Claude Code](https://claude.com/claude-code) diff --git a/docs/WINDOWS.md b/docs/WINDOWS.md new file mode 100644 index 00000000..9870a50a --- /dev/null +++ b/docs/WINDOWS.md @@ -0,0 +1,122 @@ +# Запуск на ноутбуке с Windows и сценарий защиты ЛР 4 + +Команды — в **PowerShell** из папки проекта (где лежит `compose.yaml`). Переменные `$env:...` действуют до закрытия окна. + +## 1. Что установить один раз + +| Программа | Зачем | Важно | +| --- | --- | --- | +| [Git for Windows](https://git-scm.com/download/win) | clone и Git Bash | путь по умолчанию `C:\Program Files\Git`: оттуда `scripts\*.cmd` запускают `.sh`-хуки | +| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | PostgreSQL, Valkey, SeaweedFS (S3), HAProxy, API | backend WSL 2, виртуализация в BIOS; в Settings → Resources ≥ 4 CPU и ≥ 6 ГБ RAM | +| [Python 3.12+](https://www.python.org/downloads/windows/) | secrets, публичные и свои тесты, скрипты замеров | галочка **Add python.exe to PATH** | +| GNU make (необязательно) | `make check/test/test-failover …` | `winget install ezwinports.make`, затем **новое** окно PowerShell. Рецептам Makefile нужен `sh`: в каждом окне один раз `$env:Path += ";C:\Program Files\Git\usr\bin"` | + +## 2. Развернуть проект (дома, заранее — нужен интернет) + +```powershell +git clone https://gitea.privcloud.tech/wfa/mtusi2026lab4 mtusi2026lab4 +cd mtusi2026lab4 +git checkout solution/bvt2302-bodanyuk +python scripts\init_secrets.py # случайные пароли и 3 пары ключей S3 в .\secrets (не в Git) +Copy-Item .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +``` + +Первый запуск качает образы (~1 ГБ) и собирает API — 3–10 минут. Команда завершается, когда postgres, valkey, s3, обе реплики api и lb в состоянии `healthy`, а `migrate` и `s3-init` завершились с кодом 0. Вердикты bootstrap S3 (приватность, права ключей): `docker compose logs s3-init`. + +Окружение для своих тестов и скриптов (один раз): + +```powershell +python -m venv .venv +.venv\Scripts\python -m pip install --require-hashes --no-deps -r requirements.txt +.venv\Scripts\python -m pip install -r requirements-test.txt -r requirements-dev.txt +$env:PYTHONUTF8 = "1" +``` + +Если запущен стенд lab3, он занимает порт 8080: `docker compose -p mtusi-lab3 stop` (данные lab3 сохранятся). + +## 3. Перед защитой + +```powershell +cd mtusi2026lab4 +$env:PYTHONUTF8 = "1"; $env:Path += ";C:\Program Files\Git\usr\bin" +docker compose up -d --wait +make check test PYTHON=python # или: python tests\smoke.py +python scripts\lbctl.py status # обе реплики UP +``` + +## 4. Сценарий защиты из README — по шагам + +**Шаг 1. Загрузить fixture, отправить сообщение, скачать вторым участником, сравнить SHA-256.** + +```powershell +.venv\Scripts\python scripts\demo_attachments.py +``` + +Скрипт печатает каждый запрос и ответ: регистрирует alice, bob и eve, создаёт чат alice+bob, загружает `tests/fixtures/sample.png` (201 ready, sha256), отправляет сообщение с `attachment_ids` и `text=""`. Затем bob скачивает файл, SHA-256 совпадает. Для eve (посторонний) — 404 на метаданные и на байты. Отдельно показано, что ключ объекта в S3 случайный и в API не раскрывается. + +**Шаг 2. Прямой неаутентифицированный GET к S3 не отдаёт объект.** Скрипт из шага 1 в конце делает изнутри сети `storage` (снаружи S3 не опубликован) анонимный GET того же объекта и анонимный листинг bucket — оба 403. Тот же GET, подписанный ключом API, — 200. Вручную: + +```powershell +docker compose logs s3-init # anonymous GET/LIST -> 403, права ключей api/reconciler +.venv\Scripts\python -m pytest student-tests\test_storage_failures.py -k private -v +``` + +**Шаг 3. Persistence и failover: общий тест сохраняет и заново читает оригинал через API.** + +```powershell +make test-persistence ACTION_SCRIPT=scripts/restart.cmd PYTHON=python # restart lb, api, postgres, valkey, s3 +docker compose ps # у всех «Up N seconds» +make test-failover ACTION_SCRIPT=scripts/stop-one.cmd PYTHON=python +python scripts\lbctl.py replicas # остановленная — exited +make restore PYTHON=python +``` + +Для второй реплики: сдвинуть round-robin одним запросом (`Invoke-WebRequest http://127.0.0.1:8080/health/live -UseBasicParsing | Out-Null`) и повторить `make test-failover`. Загрузка → kill обслужившей реплики → скачивание через другую: `.venv\Scripts\python -m pytest student-tests\test_storage_failures.py -k failover -v`. + +**Шаг 4. На API нет локального файла оригинала.** + +```powershell +.venv\Scripts\python -m pytest student-tests\test_attachments.py -k not_stored -v # find по ФС реплик +docker compose config --format json | Select-String '"volumes"' # у api нет volume +``` + +**Уровень 4 вживую.** + +```powershell +# Отказ S3 (stop и pause): readiness 503, загрузка 503 за < 8 с, текст работает (деградированный режим LB): +.venv\Scripts\python -m pytest student-tests\test_storage_failures.py -k outage -v +# Сбой между PUT и COMMIT убирает janitor; потерянный ответ после COMMIT ничего не удаляет: +.venv\Scripts\python -m pytest student-tests\test_consistency.py -v +# Все негативные кейсы файлов (413/415/400, чужой чат, MIME): +.venv\Scripts\python -m pytest student-tests\test_attachments.py -v +``` + +**Трек 2 вживую.** + +```powershell +docker compose run --rm reconciler # dry-run: отчёт JSON, ничего не меняет +.venv\Scripts\python -m pytest student-tests\test_reconcile.py -k both_sides -v -s # ошибки на обеих сторонах, ~1,5 мин +docker compose run --rm reconciler --apply # исправить (на чистом стенде — 0 действий) +``` + +Готовые отчёты прогонов — [evidence/reconcile/](../evidence/reconcile/), стоимость обхода — [evidence/reconcile/cost.json](../evidence/reconcile/cost.json). + +**Все свои тесты** (~25 мин): `.venv\Scripts\python -m pytest student-tests -v`. + +## 5. Если что-то пошло не так + +| Симптом | Что сделать | +| --- | --- | +| `error during connect … dockerDesktopLinuxEngine` | Docker Desktop не запущен — запустить и подождать минуту | +| `port is already allocated` на 8080/8404 | занято стендом lab3 (`docker compose -p mtusi-lab3 stop`) или в `.env` поставить `APP_PUBLISH_PORT=8081`, `LB_STATS_PORT=8405` | +| `secrets/... not found` при `up` | не выполнен `python scripts\init_secrets.py` | +| `s3-init` exit 1 | `docker compose logs s3-init`: какая проверка доступа не прошла | +| `UnicodeDecodeError` в `tests\*.py` | не задан `$env:PYTHONUTF8 = "1"` | +| `*.cmd`: «не удаётся найти путь» | Git установлен не в `C:\Program Files\Git` — поправить путь в `scripts\*.cmd` | +| после `stop-one` в LB одна реплика | так и задумано; вернуть: `make restore PYTHON=python` | +| api `unhealthy` после сна ноутбука | `docker compose restart` и `python scripts\lbctl.py wait-up 2` | +| нужно начать с чистого листа | `docker compose down -v` (удаляет учебные данные, включая bucket) → `docker compose up -d --build --wait` | + +Остановить стенд после защиты: `docker compose down` (данные сохранятся) или `docker compose down -v` (данные удаляются). diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 00000000..4bce11d5 --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,294 @@ +# Clean clone: secrets, config, build, migrations from zero, start, versions +# 2026-10-25T11:39:46Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов + +$ git status --short --branch +## solution/bvt2302-bodanyuk...origin/solution/bvt2302-bodanyuk +[exit 0, 0.1 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe --version +Python 3.14.6 +[exit 0, 0.0 s] + +$ docker version --format client {{.Client.Version}} / server {{.Server.Version}} +client 29.8.0 / server 29.8.0 +[exit 0, 0.1 s] + +$ docker compose version +Docker Compose version v5.5.1 +[exit 0, 0.1 s] + +$ make --version +GNU Make 4.4.1 +Built for Windows32 +Copyright (C) 1988-2023 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +[exit 0, 0.1 s] + +$ python scripts/init_secrets.py +created secrets/postgres_password +created secrets/valkey_password +created secrets/database_url +created secrets/cache_url +created secrets/s3_admin_access_key_id +created secrets/s3_admin_secret_access_key +created secrets/s3_api_access_key_id +created secrets/s3_api_secret_access_key +created secrets/s3_reconciler_access_key_id +created secrets/s3_reconciler_secret_access_key +[exit 0, 0.1 s] + +$ cp .env.example .env + +$ docker compose --env-file .env config --quiet + +[exit 0, 0.2 s] + +$ docker compose --env-file .env up -d --build --wait +#1 [internal] load local bake definitions +#1 reading from stdin 1.32kB 0.0s done +#1 DONE 0.0s + +#2 [migrate internal] load build definition from Dockerfile +#2 transferring dockerfile: 2.90kB done +#2 DONE 0.0s + +#3 [api] resolve image config for docker-image://docker.io/docker/dockerfile:1 +#3 DONE 1.1s + +#4 [migrate] docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e +#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.0s done +#4 CACHED + +#5 [api internal] load metadata for docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 +#5 DONE 0.0s + +#6 [api internal] load .dockerignore +#6 transferring context: 196B done +#6 DONE 0.0s + +#7 [migrate 1/9] FROM docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 +#7 resolve docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 0.0s done +#7 DONE 0.1s + +#8 [migrate internal] load build context +#8 transferring context: 420.34kB 0.1s done +#8 transferring context: 420.34kB 0.1s done +#8 DONE 0.1s + +#9 [api 2/9] WORKDIR /app +#9 CACHED + +#10 [api 5/9] RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 && rm -rf /var/lib/apt/lists/* +#10 CACHED + +#11 [api 3/9] COPY requirements.txt ./ +#11 CACHED + +#12 [api 4/9] RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt +#12 CACHED + +#13 [api 7/9] COPY src/ ./src/ +#13 CACHED + +#14 [api 6/9] RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app +#14 CACHED + +#15 [api 8/9] COPY migrations/ ./migrations/ +#15 CACHED + +#16 [api 9/9] COPY contracts/openapi.json ./contracts/openapi.json +#16 DONE 0.0s + +#17 [api] exporting to image +#17 exporting layers +#17 exporting layers 0.1s done +#17 exporting manifest sha256:61c866ef0176db37bf16836248289c639db9adf4af46bbd1bbd6b5bf36f37d64 0.0s done +#17 exporting config sha256:51199f0c44f5a7bb410bb7446b9c1a569b46165515f46d4065fb1fcdb4bbeb73 0.0s done +#17 exporting attestation manifest sha256:05366afe97838405c3e894464a12615f917115bec85c16befdc5503003c1f470 +#17 ... + +#18 [migrate] exporting to image +#18 exporting layers 0.1s done +#18 exporting manifest sha256:4110f2c418eaa8d2334845ca16721650556be9cf998b3763e558d1677128333e 0.0s done +#18 exporting config sha256:fdc58a5045cb098687769b14073cec7a7867599a2eb441679c0ac97df186c22b 0.0s done +#18 exporting attestation manifest sha256:d356c4ccd69e747feceab14426981048b02a2b0bcba8a29ad66d24ffe553177c 0.0s done +#18 exporting manifest list sha256:cf5183dbd465657100545822d97a395adee71d05ac7e2b9d5f5f4fa25de424dc 0.0s done +#18 naming to docker.io/library/mtusi-messenger:lab4 done +#18 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done +#18 DONE 0.3s + +#17 [api] exporting to image +#17 exporting attestation manifest sha256:05366afe97838405c3e894464a12615f917115bec85c16befdc5503003c1f470 0.0s done +#17 exporting manifest list sha256:c438a3f95cfa71ecc8ddb2d7bda0b2f7a4b549ba73a3a45c523539d162c8f2de 0.0s done +#17 naming to docker.io/library/mtusi-messenger:lab4 done +#17 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done +#17 DONE 0.4s + +#19 [migrate] resolving provenance for metadata file +#19 DONE 0.0s + +#20 [api] resolving provenance for metadata file +#20 DONE 0.0s + Image mtusi-messenger:lab4 Building + Image mtusi-messenger:lab4 Building + Image mtusi-messenger:lab4 Built + Image mtusi-messenger:lab4 Built + Network mtusi-lab4_edge Creating + Network mtusi-lab4_data Creating + Volume mtusi-lab4_s3data Creating + Network mtusi-lab4_edge Creating + Volume mtusi-lab4_pgdata Creating + Volume mtusi-lab4_pgdata Creating + Network mtusi-lab4_storage Creating + Network mtusi-lab4_storage Creating + Volume mtusi-lab4_s3data Creating + Network mtusi-lab4_data Creating + Volume mtusi-lab4_valkeydata Creating + Volume mtusi-lab4_valkeydata Creating + Volume mtusi-lab4_s3data Created + Volume mtusi-lab4_s3data Created + Volume mtusi-lab4_pgdata Created + Volume mtusi-lab4_pgdata Created + Volume mtusi-lab4_valkeydata Created + Volume mtusi-lab4_valkeydata Created + Network mtusi-lab4_edge Created + Network mtusi-lab4_edge Created + Network mtusi-lab4_storage Created + Network mtusi-lab4_storage Created + Container mtusi-lab4-s3-1 Creating + Network mtusi-lab4_data Created + Network mtusi-lab4_data Created + Container mtusi-lab4-valkey-1 Creating + Container mtusi-lab4-postgres-1 Creating + Container mtusi-lab4-valkey-1 Created + Container mtusi-lab4-postgres-1 Created + Container mtusi-lab4-migrate-1 Creating + Container mtusi-lab4-s3-1 Created + Container mtusi-lab4-migrate-1 Created + Container mtusi-lab4-s3-init-1 Creating + Container mtusi-lab4-s3-init-1 Created + Container mtusi-lab4-api-2 Creating + Container mtusi-lab4-api-1 Creating + Container mtusi-lab4-api-2 Created + Container mtusi-lab4-lb-1 Creating + Container mtusi-lab4-lb-1 Created + Container mtusi-lab4-api-1 Created + Container mtusi-lab4-postgres-1 Starting + Container mtusi-lab4-s3-1 Starting + Container mtusi-lab4-valkey-1 Starting + Container mtusi-lab4-postgres-1 Started + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-1 Started + Container mtusi-lab4-valkey-1 Started + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-s3-init-1 Starting + Container mtusi-lab4-s3-init-1 Started + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-valkey-1 Waiting + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-s3-init-1 Waiting + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-valkey-1 Healthy + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-s3-init-1 Exited + Container mtusi-lab4-api-1 Starting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Starting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-lb-1 Starting + Container mtusi-lab4-lb-1 Started + Container mtusi-lab4-s3-init-1 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-lb-1 Waiting + Container mtusi-lab4-valkey-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-valkey-1 Healthy + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-s3-init-1 Exited + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-lb-1 Healthy +[exit 0, 31.3 s] + +$ docker compose ps -a --format table {{.Service}} {{.Image}} {{.Status}} +SERVICE IMAGE STATUS +api mtusi-messenger:lab4 Up 12 seconds (healthy) +api mtusi-messenger:lab4 Up 12 seconds (healthy) +lb haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 Up 5 seconds (healthy) +migrate mtusi-messenger:lab4 Exited (0) 18 seconds ago +postgres postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c Up 25 seconds (healthy) +s3 chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 Up 25 seconds (healthy) +s3-init mtusi-messenger:lab4 Exited (0) 12 seconds ago +valkey valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 Up 25 seconds (healthy) +[exit 0, 0.3 s] + +$ docker compose logs --no-log-prefix migrate +migrate: 0001_init: applied +migrate: 0002_idempotency_keys: applied +migrate: 0003_post_message_function: applied +migrate: 0004_attachments: applied +migrate: done, 4 migration file(s) checked +[exit 0, 0.3 s] + +$ docker compose logs --no-log-prefix s3-init +s3-bootstrap: bucket messenger-attachments: created +s3-bootstrap: sentinel health/sentinel: written +s3-bootstrap: anonymous GET object: HTTP 403 -> refused +s3-bootstrap: anonymous LIST bucket: HTTP 403 -> refused +s3-bootstrap: api: read object: allowed -> OK +s3-bootstrap: api: write object: allowed -> OK +s3-bootstrap: api: delete object: allowed -> OK +s3-bootstrap: api: list bucket: refused (AccessDenied) -> OK +s3-bootstrap: api: create bucket: refused (AccessDenied) -> OK +s3-bootstrap: api: read object of another bucket: refused (403) -> OK +s3-bootstrap: reconciler: list bucket: allowed -> OK +s3-bootstrap: reconciler: create bucket: refused (AccessDenied) -> OK +s3-bootstrap: api: buckets visible in ListBuckets: [] -> OK +s3-bootstrap: done: private bucket, least-privilege keys verified +[exit 0, 0.3 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py replicas +mtusi-lab4-api-1 X-Instance-Id=8691c30aa702 running health=healthy pid=28760 started=2026-10-25T11:40:06.635 ip=172.25.0.2,172.26.0.3,172.27.0.4 +mtusi-lab4-api-2 X-Instance-Id=4026e296947c running health=healthy pid=28857 started=2026-10-25T11:40:07.085 ip=172.25.0.3,172.26.0.4,172.27.0.5 +[exit 0, 0.5 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status +api1 172.25.0.2 UP mtusi-lab4-api-1 X-Instance-Id=8691c30aa702 +api2 172.25.0.3 UP mtusi-lab4-api-2 X-Instance-Id=4026e296947c +(other slots: MAINT, no container) +[exit 0, 0.7 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe -m venv .venv + +[exit 0, 6.9 s] + +$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install -q --require-hashes --no-deps -r requirements.txt + +[notice] A new release of pip is available: 26.1.2 -> 26.2.2 +[notice] To update, run: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install --upgrade pip +[exit 0, 21.2 s] + +$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install -q -r requirements-test.txt -r requirements-dev.txt + +[notice] A new release of pip is available: 26.1.2 -> 26.2.2 +[notice] To update, run: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install --upgrade pip +[exit 0, 14.0 s] diff --git a/evidence/final-commit-public-checks.txt b/evidence/final-commit-public-checks.txt new file mode 100644 index 00000000..129369ae --- /dev/null +++ b/evidence/final-commit-public-checks.txt @@ -0,0 +1,16 @@ +# Public checks repeated on the final commit fc50362 after the fixes (2026-10-25T12:34:34Z) +$ make test +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +$ make test-persistence ACTION_SCRIPT=scripts/restart.cmd +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +2 API replica(s) UP in HAProxy after 0.7 s: ['0e899ebdfc56', '92fdfb319248'] +$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 5.837} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab4-api-2 X-Instance-Id=0e899ebdfc56 exited health=unhealthy pid=0 started=2026-10-25T12:35:22.631 ip=- +2 API replica(s) UP in HAProxy after 0.6 s: ['0e899ebdfc56', '92fdfb319248'] +$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 5.922} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab4-api-1 X-Instance-Id=92fdfb319248 exited health=unhealthy pid=0 started=2026-10-25T12:35:22.981 ip=- +2 API replica(s) UP in HAProxy after 0.6 s: ['0e899ebdfc56', '92fdfb319248'] diff --git a/evidence/lab4-runs.txt b/evidence/lab4-runs.txt new file mode 100644 index 00000000..535b56b8 --- /dev/null +++ b/evidence/lab4-runs.txt @@ -0,0 +1,328 @@ +# Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py +# 2026-10-25T12:03:36Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов + +$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/upload_memory.py --output evidence/memory +generating files ... +{"scenario": "png-9.4MiB x16, slots=4 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 16, "s3_sampled_max_mib": 548.3, "s3_anon_after_mib": 386.7, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 16}, "error_reasons": {}, "wall_seconds": 4.37, "latency_s": {"p50": 3.62, "max": 4.34}, "replicas": {"f2c108a7e1b2": {"idle_mib": 75.3, "peak_mib": 156.3, "sampled_max_mib": 156.1, "python_vmhwm_mib": 152.6, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 81.0}, "2e1bacf102e6": {"idle_mib": 74.5, "peak_mib": 163.2, "sampled_max_mib": 161.7, "python_vmhwm_mib": 152.9, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 88.7}}} +{"scenario": "png-9.4MiB x32, slots=4 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 32, "s3_sampled_max_mib": 800.5, "s3_anon_after_mib": 399.1, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 25, "503": 7}, "error_reasons": {"Слишком много одновременных загрузок, повторите позже": 7}, "wall_seconds": 6.96, "latency_s": {"p50": 4.99, "max": 6.91}, "replicas": {"5e7cbb39228d": {"idle_mib": 74.9, "peak_mib": 155.1, "sampled_max_mib": 153.6, "python_vmhwm_mib": 143.8, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 80.2}, "0036ead083c5": {"idle_mib": 74.6, "peak_mib": 187.5, "sampled_max_mib": 185.9, "python_vmhwm_mib": 181.2, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 112.9}}} +{"scenario": "png-9.4MiB x32, slots=16 (bound raised)", "settings": {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 32, "s3_sampled_max_mib": 1023.9, "s3_anon_after_mib": 411.8, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 24, "503": 8}, "error_reasons": {"Хранилище файлов временно недоступно": 8}, "wall_seconds": 10.2, "latency_s": {"p50": 7.46, "max": 10.04}, "replicas": {"04b50b0bdc7f": {"idle_mib": 75.2, "peak_mib": 284.6, "sampled_max_mib": 284.3, "python_vmhwm_mib": 285.9, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 209.4}, "f860d892caaf": {"idle_mib": 75.1, "peak_mib": 268.2, "sampled_max_mib": 265.8, "python_vmhwm_mib": 266.7, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 193.1}}} +{"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=1 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 16, "s3_sampled_max_mib": 990.1, "s3_anon_after_mib": 386.0, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 0.6, "types": ["image/jpeg"], "status_codes": {"201": 16}, "error_reasons": {}, "wall_seconds": 5.77, "latency_s": {"p50": 3.58, "max": 5.74}, "replicas": {"2c78d3ce98f5": {"idle_mib": 75.0, "peak_mib": 317.1, "sampled_max_mib": 315.5, "python_vmhwm_mib": 107.5, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 242.1}, "2220bc508f5c": {"idle_mib": 74.7, "peak_mib": 316.5, "sampled_max_mib": 315.5, "python_vmhwm_mib": 107.2, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 241.8}}} +{"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=4", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, "clients": 16, "s3_sampled_max_mib": 995.3, "s3_anon_after_mib": 386.6, "s3_limit_mib": 1024.0, "incidents": {"mtusi-lab4-api-1": "restarts=0 oom_kill=6", "mtusi-lab4-api-2": "restarts=0 oom_kill=7"}, "file_mib_each": 0.6, "types": ["image/jpeg"], "status_codes": {"201": 2, "503": 14}, "error_reasons": {"Не хватило ресурсов для проверки изображения, повторите позже": 13, "Слишком много одновременных загрузок, повторите позже": 1}, "wall_seconds": 8.15, "latency_s": {"p50": 8.12, "max": 8.12}, "replicas": {"984d6a81b4b8": {"idle_mib": 86.2, "peak_mib": 512.0, "sampled_max_mib": 511.9, "python_vmhwm_mib": 110.7, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 425.8}, "42f112fb4559": {"idle_mib": 74.5, "peak_mib": 512.0, "sampled_max_mib": 507.9, "python_vmhwm_mib": 92.3, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 437.5}}} +{"date": "2026-10-25 15:08:28", "host": "Windows-10-10.0.19045-SP0", "cpus": 8, "commit": "f9f5ab9", "api_limits": "1000000000 536870912", "jpeg_file_mib": 0.6, "png_file_mib": 9.42} +[exit 0, 292.6 s] + +$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/reconcile_faults.py --output evidence/reconcile +{ + "dry_run_changed_nothing": true, + "per_key": { + "dry-run": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": { + "kind": "missing_object", + "result": "planned", + "used": true + }, + "unused, object truncated D": { + "kind": "size_mismatch", + "result": "planned", + "used": false + }, + "quarantined but healthy E": { + "kind": "restored_object", + "result": "planned", + "used": true + }, + "same-size corruption F": { + "kind": "checksum_mismatch", + "result": "planned", + "used": false + }, + "old orphan object": { + "kind": "orphan_object", + "result": "planned", + "used": false + }, + "stale intent + object": { + "kind": "stale_intent", + "result": "planned", + "used": false + }, + "stale intent, no object": { + "kind": "stale_intent", + "result": "planned", + "used": false + }, + "fresh orphan (in-flight)": { + "kind": "fresh_orphan_object", + "result": "not_actionable", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "fresh_intent", + "result": "not_actionable", + "used": false + } + }, + "apply": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": { + "kind": "missing_object", + "result": "applied", + "used": true + }, + "unused, object truncated D": { + "kind": "size_mismatch", + "result": "applied", + "used": false + }, + "quarantined but healthy E": { + "kind": "restored_object", + "result": "applied", + "used": true + }, + "same-size corruption F": { + "kind": "checksum_mismatch", + "result": "applied", + "used": false + }, + "old orphan object": { + "kind": "orphan_object", + "result": "applied", + "used": false + }, + "stale intent + object": { + "kind": "stale_intent", + "result": "applied", + "used": false + }, + "stale intent, no object": { + "kind": "stale_intent", + "result": "applied", + "used": false + }, + "fresh orphan (in-flight)": { + "kind": "fresh_orphan_object", + "result": "not_actionable", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "fresh_intent", + "result": "not_actionable", + "used": false + } + }, + "apply again": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": null, + "unused, object truncated D": null, + "quarantined but healthy E": null, + "same-size corruption F": null, + "old orphan object": null, + "stale intent + object": null, + "stale intent, no object": null, + "fresh orphan (in-flight)": { + "kind": "fresh_orphan_object", + "result": "not_actionable", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "fresh_intent", + "result": "not_actionable", + "used": false + } + }, + "after restore of C": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": { + "kind": "restored_object", + "result": "applied", + "used": true + }, + "unused, object truncated D": null, + "quarantined but healthy E": null, + "same-size corruption F": null, + "old orphan object": null, + "stale intent + object": null, + "stale intent, no object": null, + "fresh orphan (in-flight)": { + "kind": "orphan_object", + "result": "applied", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "stale_intent", + "result": "applied", + "used": false + } + } + }, + "objects_present_at_end (fresh items were older than --grace by the later runs)": { + "healthy used A": true, + "healthy unused B": true, + "used, object deleted C": true, + "unused, object truncated D": true, + "quarantined but healthy E": true, + "same-size corruption F": true, + "old orphan object": false, + "stale intent + object": false, + "stale intent, no object": false, + "fresh orphan (in-flight)": false, + "fresh intent (in-flight)": false + }, + "rows_after": { + "C": { + "status": "ready", + "lost_reason": null + }, + "D": { + "status": "ready", + "lost_reason": "size_mismatch" + }, + "E": { + "status": "ready", + "lost_reason": null + }, + "F": { + "status": "ready", + "lost_reason": "checksum_mismatch" + }, + "A": { + "status": "ready", + "lost_reason": null + }, + "B": { + "status": "ready", + "lost_reason": null + } + }, + "content_status_after": { + "A": 200, + "B": 200, + "C": 200, + "D": 404, + "E": 200, + "F": 404 + }, + "durations_s": [ + 3.94, + 3.983, + 3.0, + 3.548 + ] +} +{ + "orphans_injected": 2000, + "interrupted_run": { + "exit_code": 1, + "error": "store_unavailable: Unavailable", + "applied": 440 + }, + "orphans_left_after_interrupt": 1560, + "rerun": { + "exit_code": 0, + "applied": 1562, + "error": null + }, + "third": { + "exit_code": 0, + "planned": 0 + }, + "orphans_left_at_end": 0 +} +[exit 0, 182.1 s] + +$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/reconcile_cost.py --output evidence/reconcile +removed 0 synthetic attachments +Traceback (most recent call last): + File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 155, in + sys.exit(main()) + ~~~~^^ + File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 127, in main + add(target - present, owner, chat) + ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 89, in add + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + "object_key, status, created_at, updated_at) " + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, " + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', " + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 62, in psql + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v", + ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + "ON_ERROR_STOP=1", "-c", sql).stdout.strip() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 57, in compose + return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True, + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + encoding="utf-8", timeout=timeout, check=check) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 555, in run + with Popen(*popenargs, **kwargs) as process: + ~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 1039, in __init__ + self._execute_child(args, executable, preexec_fn, close_fds, + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + pass_fds, cwd, env, + ^^^^^^^^^^^^^^^^^^^ + ...<5 lines>... + gid, gids, uid, umask, + ^^^^^^^^^^^^^^^^^^^^^^ + start_new_session, process_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 1553, in _execute_child + hp, ht, pid, tid = _winapi.CreateProcess(executable, args, + ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ + # no special security + ^^^^^^^^^^^^^^^^^^^^^ + ...<4 lines>... + cwd, + ^^^^ + startupinfo) + ^^^^^^^^^^^^ +FileNotFoundError: [WinError 206] Имя файла или его расширение имеет слишком большую длину +[exit 1, 1.3 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2 +2 API replica(s) UP in HAProxy after 0.6 s: ['379d13a0775a', '862067d68c1c'] +[exit 0, 1.4 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.091, + "concurrency": 4, + "successful_requests": 5416, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 179.989, + "successful_latency_ms": { + "p50": 17.684, + "p95": 49.053, + "p99": 63.81 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +[exit 0, 30.7 s] + +$ docker compose ps -a --format table {{.Service}} {{.Status}} +SERVICE STATUS +api Up 3 minutes (healthy) +api Up 3 minutes (healthy) +lb Up 24 minutes (healthy) +migrate Exited (0) 10 minutes ago +postgres Up 11 minutes (healthy) +s3 Up 6 minutes (healthy) +s3-init Exited (0) 10 minutes ago +valkey Up 12 minutes (healthy) +[exit 0, 0.4 s] diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..fb5757f6 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.091, + "concurrency": 4, + "successful_requests": 5416, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 179.989, + "successful_latency_ms": { + "p50": 17.684, + "p95": 49.053, + "p99": 63.81 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/make-check-validate-test.txt b/evidence/make-check-validate-test.txt new file mode 100644 index 00000000..287d30c5 --- /dev/null +++ b/evidence/make-check-validate-test.txt @@ -0,0 +1,17 @@ +# make check, make validate (venv), make test +# 2026-10-25T11:41:03Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов + +$ make check +python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +[exit 0, 1.3 s] + +$ make validate PYTHON=.venv/Scripts/python.exe +.venv/Scripts/python.exe scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +[exit 0, 2.3 s] + +$ make test +python tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +[exit 0, 1.3 s] diff --git a/evidence/memory/summary.json b/evidence/memory/summary.json new file mode 100644 index 00000000..a862aa7c --- /dev/null +++ b/evidence/memory/summary.json @@ -0,0 +1,251 @@ +{ + "meta": { + "date": "2026-10-25 15:08:28", + "host": "Windows-10-10.0.19045-SP0", + "cpus": 8, + "commit": "f9f5ab9", + "api_limits": "1000000000 536870912", + "jpeg_file_mib": 0.6, + "png_file_mib": 9.42 + }, + "scenarios": [ + { + "scenario": "png-9.4MiB x16, slots=4 (default)", + "settings": { + "UPLOAD_CONCURRENCY": "4", + "IMAGE_VERIFY_CONCURRENCY": "1" + }, + "clients": 16, + "s3_sampled_max_mib": 548.3, + "s3_anon_after_mib": 386.7, + "s3_limit_mib": 1024.0, + "incidents": {}, + "file_mib_each": 9.42, + "types": [ + "image/png" + ], + "status_codes": { + "201": 16 + }, + "error_reasons": {}, + "wall_seconds": 4.37, + "latency_s": { + "p50": 3.62, + "max": 4.34 + }, + "replicas": { + "f2c108a7e1b2": { + "idle_mib": 75.3, + "peak_mib": 156.3, + "sampled_max_mib": 156.1, + "python_vmhwm_mib": 152.6, + "uploads_served": 8, + "limit_mib": 512.0, + "peak_minus_idle_mib": 81.0 + }, + "2e1bacf102e6": { + "idle_mib": 74.5, + "peak_mib": 163.2, + "sampled_max_mib": 161.7, + "python_vmhwm_mib": 152.9, + "uploads_served": 8, + "limit_mib": 512.0, + "peak_minus_idle_mib": 88.7 + } + } + }, + { + "scenario": "png-9.4MiB x32, slots=4 (default)", + "settings": { + "UPLOAD_CONCURRENCY": "4", + "IMAGE_VERIFY_CONCURRENCY": "1" + }, + "clients": 32, + "s3_sampled_max_mib": 800.5, + "s3_anon_after_mib": 399.1, + "s3_limit_mib": 1024.0, + "incidents": {}, + "file_mib_each": 9.42, + "types": [ + "image/png" + ], + "status_codes": { + "201": 25, + "503": 7 + }, + "error_reasons": { + "Слишком много одновременных загрузок, повторите позже": 7 + }, + "wall_seconds": 6.96, + "latency_s": { + "p50": 4.99, + "max": 6.91 + }, + "replicas": { + "5e7cbb39228d": { + "idle_mib": 74.9, + "peak_mib": 155.1, + "sampled_max_mib": 153.6, + "python_vmhwm_mib": 143.8, + "uploads_served": 16, + "limit_mib": 512.0, + "peak_minus_idle_mib": 80.2 + }, + "0036ead083c5": { + "idle_mib": 74.6, + "peak_mib": 187.5, + "sampled_max_mib": 185.9, + "python_vmhwm_mib": 181.2, + "uploads_served": 16, + "limit_mib": 512.0, + "peak_minus_idle_mib": 112.9 + } + } + }, + { + "scenario": "png-9.4MiB x32, slots=16 (bound raised)", + "settings": { + "UPLOAD_CONCURRENCY": "16", + "IMAGE_VERIFY_CONCURRENCY": "1" + }, + "clients": 32, + "s3_sampled_max_mib": 1023.9, + "s3_anon_after_mib": 411.8, + "s3_limit_mib": 1024.0, + "incidents": {}, + "file_mib_each": 9.42, + "types": [ + "image/png" + ], + "status_codes": { + "201": 24, + "503": 8 + }, + "error_reasons": { + "Хранилище файлов временно недоступно": 8 + }, + "wall_seconds": 10.2, + "latency_s": { + "p50": 7.46, + "max": 10.04 + }, + "replicas": { + "04b50b0bdc7f": { + "idle_mib": 75.2, + "peak_mib": 284.6, + "sampled_max_mib": 284.3, + "python_vmhwm_mib": 285.9, + "uploads_served": 16, + "limit_mib": 512.0, + "peak_minus_idle_mib": 209.4 + }, + "f860d892caaf": { + "idle_mib": 75.1, + "peak_mib": 268.2, + "sampled_max_mib": 265.8, + "python_vmhwm_mib": 266.7, + "uploads_served": 16, + "limit_mib": 512.0, + "peak_minus_idle_mib": 193.1 + } + } + }, + { + "scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=1 (default)", + "settings": { + "UPLOAD_CONCURRENCY": "4", + "IMAGE_VERIFY_CONCURRENCY": "1" + }, + "clients": 16, + "s3_sampled_max_mib": 990.1, + "s3_anon_after_mib": 386.0, + "s3_limit_mib": 1024.0, + "incidents": {}, + "file_mib_each": 0.6, + "types": [ + "image/jpeg" + ], + "status_codes": { + "201": 16 + }, + "error_reasons": {}, + "wall_seconds": 5.77, + "latency_s": { + "p50": 3.58, + "max": 5.74 + }, + "replicas": { + "2c78d3ce98f5": { + "idle_mib": 75.0, + "peak_mib": 317.1, + "sampled_max_mib": 315.5, + "python_vmhwm_mib": 107.5, + "uploads_served": 8, + "limit_mib": 512.0, + "peak_minus_idle_mib": 242.1 + }, + "2220bc508f5c": { + "idle_mib": 74.7, + "peak_mib": 316.5, + "sampled_max_mib": 315.5, + "python_vmhwm_mib": 107.2, + "uploads_served": 8, + "limit_mib": 512.0, + "peak_minus_idle_mib": 241.8 + } + } + }, + { + "scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=4", + "settings": { + "UPLOAD_CONCURRENCY": "4", + "IMAGE_VERIFY_CONCURRENCY": "4" + }, + "clients": 16, + "s3_sampled_max_mib": 995.3, + "s3_anon_after_mib": 386.6, + "s3_limit_mib": 1024.0, + "incidents": { + "mtusi-lab4-api-1": "restarts=0 oom_kill=6", + "mtusi-lab4-api-2": "restarts=0 oom_kill=7" + }, + "file_mib_each": 0.6, + "types": [ + "image/jpeg" + ], + "status_codes": { + "201": 2, + "503": 14 + }, + "error_reasons": { + "Не хватило ресурсов для проверки изображения, повторите позже": 13, + "Слишком много одновременных загрузок, повторите позже": 1 + }, + "wall_seconds": 8.15, + "latency_s": { + "p50": 8.12, + "max": 8.12 + }, + "replicas": { + "984d6a81b4b8": { + "idle_mib": 86.2, + "peak_mib": 512.0, + "sampled_max_mib": 511.9, + "python_vmhwm_mib": 110.7, + "uploads_served": 8, + "limit_mib": 512.0, + "peak_minus_idle_mib": 425.8 + }, + "42f112fb4559": { + "idle_mib": 74.5, + "peak_mib": 512.0, + "sampled_max_mib": 507.9, + "python_vmhwm_mib": 92.3, + "uploads_served": 8, + "limit_mib": 512.0, + "peak_minus_idle_mib": 437.5 + } + } + } + ] +} \ No newline at end of file diff --git a/evidence/reconcile-fix-rerun.txt b/evidence/reconcile-fix-rerun.txt new file mode 100644 index 00000000..b89b1eac --- /dev/null +++ b/evidence/reconcile-fix-rerun.txt @@ -0,0 +1,14 @@ +# Reconciler fix 6fb70de (checksum quarantine lifted only with --verify-checksums), 2026-10-25T12:47:16Z +$ docker compose run --rm reconciler --apply (plain pass right after quarantining 3 leftover rows with --verify-checksums) +findings {'fresh_intent': 2} actions {'planned': 0, 'applied': 0, 'skipped': 0, 'failed': 0} +$ python -m pytest student-tests/test_reconcile.py -v +student-tests/test_reconcile.py::test_detect_and_safely_fix_faults_on_both_sides PASSED [ 16%] +student-tests/test_reconcile.py::test_checksum_mode_finds_same_size_corruption PASSED [ 33%] +student-tests/test_reconcile.py::test_safety_valve_refuses_mass_deletion PASSED [ 50%] +student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[s3] PASSED [ 66%] +student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[postgres] PASSED [ 83%] +student-tests/test_reconcile.py::test_live_uploads_are_never_hurt_by_concurrent_apply PASSED [100%] +======================== 6 passed in 386.80s (0:06:26) ======================== + +# make test on 6fb70de (2026-10-25T12:54:03Z) +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/reconcile/cost.json b/evidence/reconcile/cost.json new file mode 100644 index 00000000..cd726ee2 --- /dev/null +++ b/evidence/reconcile/cost.json @@ -0,0 +1,139 @@ +{ + "meta": { + "date": "2026-10-25 15:45:05", + "host": "Windows-10-10.0.19045-SP0", + "cpus": 8, + "commit": "fc50362", + "object_bytes": 1024, + "reconciler_limits": "cpus 1.0, memory 256m (compose)", + "note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)" + }, + "results": [ + { + "synthetic": 1000, + "label": "dry-run, page 1000", + "args": [], + "runs": 3, + "duration_s_median": 0.301, + "duration_s_all": [ + 0.353, + 0.289, + 0.301 + ], + "objects": 1285, + "rows": 1286, + "findings": { + "fresh_intent": 2 + }, + "cost": { + "s3_list_requests": 2, + "s3_head_requests": 0, + "s3_get_bytes": 0, + "db_pages": 2, + "page_size": 1000 + } + }, + { + "synthetic": 10000, + "label": "dry-run, page 1000", + "args": [], + "runs": 3, + "duration_s_median": 1.669, + "duration_s_all": [ + 1.669, + 1.744, + 1.653 + ], + "objects": 10285, + "rows": 10286, + "findings": { + "fresh_intent": 2 + }, + "cost": { + "s3_list_requests": 11, + "s3_head_requests": 0, + "s3_get_bytes": 0, + "db_pages": 11, + "page_size": 1000 + } + }, + { + "synthetic": 10000, + "label": "dry-run, page 100", + "args": [ + "--page-size", + "100" + ], + "runs": 3, + "duration_s_median": 2.278, + "duration_s_all": [ + 2.278, + 1.999, + 2.312 + ], + "objects": 10285, + "rows": 10286, + "findings": { + "fresh_intent": 2 + }, + "cost": { + "s3_list_requests": 103, + "s3_head_requests": 0, + "s3_get_bytes": 0, + "db_pages": 103, + "page_size": 100 + } + }, + { + "synthetic": 10000, + "label": "dry-run, --verify-checksums", + "args": [ + "--verify-checksums" + ], + "runs": 3, + "duration_s_median": 48.411, + "duration_s_all": [ + 53.483, + 48.411, + 48.288 + ], + "objects": 10285, + "rows": 10286, + "findings": { + "checksum_mismatch": 3, + "fresh_intent": 2 + }, + "cost": { + "s3_list_requests": 11, + "s3_head_requests": 0, + "s3_get_bytes": 694445075, + "db_pages": 11, + "page_size": 1000 + } + }, + { + "synthetic": 50000, + "label": "dry-run, page 1000", + "args": [], + "runs": 3, + "duration_s_median": 8.435, + "duration_s_all": [ + 8.538, + 8.435, + 8.139 + ], + "objects": 50285, + "rows": 50286, + "findings": { + "fresh_intent": 2 + }, + "cost": { + "s3_list_requests": 51, + "s3_head_requests": 0, + "s3_get_bytes": 0, + "db_pages": 51, + "page_size": 1000 + } + } + ] +} \ No newline at end of file diff --git a/evidence/reconcile/faults-1-dry-run.json b/evidence/reconcile/faults-1-dry-run.json new file mode 100644 index 00000000..4ea51900 --- /dev/null +++ b/evidence/reconcile/faults-1-dry-run.json @@ -0,0 +1,188 @@ +{ + "mode": "dry-run", + "grace_seconds": 30.0, + "verify_checksums": true, + "duration_seconds": 3.94, + "error": null, + "scanned": { + "objects": 198, + "rows": 199, + "consistent": 181 + }, + "findings": { + "orphan_object": 1, + "stale_intent": 12, + "missing_object": 1, + "size_mismatch": 1, + "checksum_mismatch": 2, + "restored_object": 1, + "fresh_orphan_object": 1, + "fresh_intent": 1, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 2, + "actions": { + "planned": 18, + "applied": 0, + "skipped": 0, + "failed": 0 + }, + "cost": { + "s3_list_requests": 1, + "s3_head_requests": 0, + "s3_get_bytes": 668556368, + "db_pages": 1, + "page_size": 1000 + }, + "samples": { + "stale_intent": [ + { + "key": "o/02e47f043e4a2d0fdfb8823308b5a6f8", + "attachment_id": "445129ca-5e0a-48eb-9e9a-cb841aa3c21f", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/3738b895df9e4a2af8e83d5c31cbdb97", + "attachment_id": "418dcc4d-a2e9-4653-8209-5a8b77f6cbba", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/3f04e9b122374a366667ec50283ff028", + "attachment_id": "7832128b-d4b0-4128-92d0-fa4d9e998789", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/58646a726a50106dfeb7b214a0b68e45", + "attachment_id": "8919db52-4396-46c5-99cd-13df371436fc", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/5ca75dbe045e57c0683a60a2da3b4801", + "attachment_id": "53062ac4-cfbb-4393-944c-a2931ca2e34d", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/74c65b3fa98dc631e6f57909079c4da3", + "attachment_id": "a71a1c20-f815-49a7-945d-e45bdd4afec3", + "used": false, + "result": "planned", + "detail": "object absent" + }, + { + "key": "o/b65a274dcfc743b9b07ac223e9846546", + "attachment_id": "723cb122-1fd3-41e5-8e7f-712e39066ae3", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/c412437db6184a939bed9283b11a94f4", + "attachment_id": "58548b0c-4025-47c7-8377-28b85bfc1d46", + "used": false, + "result": "planned", + "detail": "object absent" + }, + { + "key": "o/c5326823b48696d02dd38321e5ddedde", + "attachment_id": "cbe8966a-bb9e-49f5-8799-fdb341929f15", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/cf65354466733e9171737583f869c0fc", + "attachment_id": "71b2e4a6-615e-4d6c-a35c-34ac0f703662", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/d3dca5b6df5b097cfd537a6b3ffd3deb", + "attachment_id": "ab9acee5-a57a-4698-8724-cd08431fa502", + "used": false, + "result": "planned", + "detail": "object present" + }, + { + "key": "o/e8cda21ebf7a465d6687da4c0b3c7a2a", + "attachment_id": "99d11e03-ae13-4255-b9a9-8ee8679024ff", + "used": false, + "result": "planned", + "detail": "object present" + } + ], + "fresh_orphan_object": [ + { + "key": "o/04232d5e57374a09b5e5368916176439", + "attachment_id": null, + "used": false, + "result": "not_actionable" + } + ], + "fresh_intent": [ + { + "key": "o/2a286f02c3854017886a0b371d08f2c9", + "attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6", + "used": false, + "result": "not_actionable", + "detail": "object present" + } + ], + "size_mismatch": [ + { + "key": "o/5be370c6443c51303e893c28529376b6", + "attachment_id": "a65a12d5-9369-4322-93ce-62f8f8a142a0", + "used": false, + "result": "planned" + } + ], + "checksum_mismatch": [ + { + "key": "o/77f4fbc630a1966cb3f6d4e09c019318", + "attachment_id": "ed0ff4c7-507b-44cd-8928-622ea6e977a8", + "used": false, + "result": "planned" + }, + { + "key": "o/ee7f3d21bebb4a56572b9b1f7d2299ee", + "attachment_id": "4c72dcfe-c46f-47a0-a442-2937c722ed99", + "used": false, + "result": "planned" + } + ], + "missing_object": [ + { + "key": "o/ac353be8ed77f5c230f7be71809ce405", + "attachment_id": "89e1b8ca-e554-4691-92b0-667b46757d10", + "used": true, + "result": "planned" + } + ], + "restored_object": [ + { + "key": "o/adc1d649f5b94260bd28445fb55ac5fa", + "attachment_id": "becdb821-076e-4bff-ada3-d0bf6823cce7", + "used": true, + "result": "planned" + } + ], + "orphan_object": [ + { + "key": "o/d2b0819448ab4d06a1dca06df4fac200", + "attachment_id": null, + "used": false, + "result": "planned" + } + ] + } +} \ No newline at end of file diff --git a/evidence/reconcile/faults-2-apply.json b/evidence/reconcile/faults-2-apply.json new file mode 100644 index 00000000..7bfd5369 --- /dev/null +++ b/evidence/reconcile/faults-2-apply.json @@ -0,0 +1,188 @@ +{ + "mode": "apply", + "grace_seconds": 30.0, + "verify_checksums": true, + "duration_seconds": 3.983, + "error": null, + "scanned": { + "objects": 198, + "rows": 199, + "consistent": 181 + }, + "findings": { + "orphan_object": 1, + "stale_intent": 12, + "missing_object": 1, + "size_mismatch": 1, + "checksum_mismatch": 2, + "restored_object": 1, + "fresh_orphan_object": 1, + "fresh_intent": 1, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 2, + "actions": { + "planned": 18, + "applied": 18, + "skipped": 0, + "failed": 0 + }, + "cost": { + "s3_list_requests": 1, + "s3_head_requests": 5, + "s3_get_bytes": 668676391, + "db_pages": 1, + "page_size": 1000 + }, + "samples": { + "stale_intent": [ + { + "key": "o/02e47f043e4a2d0fdfb8823308b5a6f8", + "attachment_id": "445129ca-5e0a-48eb-9e9a-cb841aa3c21f", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/3738b895df9e4a2af8e83d5c31cbdb97", + "attachment_id": "418dcc4d-a2e9-4653-8209-5a8b77f6cbba", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/3f04e9b122374a366667ec50283ff028", + "attachment_id": "7832128b-d4b0-4128-92d0-fa4d9e998789", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/58646a726a50106dfeb7b214a0b68e45", + "attachment_id": "8919db52-4396-46c5-99cd-13df371436fc", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/5ca75dbe045e57c0683a60a2da3b4801", + "attachment_id": "53062ac4-cfbb-4393-944c-a2931ca2e34d", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/74c65b3fa98dc631e6f57909079c4da3", + "attachment_id": "a71a1c20-f815-49a7-945d-e45bdd4afec3", + "used": false, + "result": "applied", + "detail": "object absent" + }, + { + "key": "o/b65a274dcfc743b9b07ac223e9846546", + "attachment_id": "723cb122-1fd3-41e5-8e7f-712e39066ae3", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/c412437db6184a939bed9283b11a94f4", + "attachment_id": "58548b0c-4025-47c7-8377-28b85bfc1d46", + "used": false, + "result": "applied", + "detail": "object absent" + }, + { + "key": "o/c5326823b48696d02dd38321e5ddedde", + "attachment_id": "cbe8966a-bb9e-49f5-8799-fdb341929f15", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/cf65354466733e9171737583f869c0fc", + "attachment_id": "71b2e4a6-615e-4d6c-a35c-34ac0f703662", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/d3dca5b6df5b097cfd537a6b3ffd3deb", + "attachment_id": "ab9acee5-a57a-4698-8724-cd08431fa502", + "used": false, + "result": "applied", + "detail": "object present" + }, + { + "key": "o/e8cda21ebf7a465d6687da4c0b3c7a2a", + "attachment_id": "99d11e03-ae13-4255-b9a9-8ee8679024ff", + "used": false, + "result": "applied", + "detail": "object present" + } + ], + "fresh_orphan_object": [ + { + "key": "o/04232d5e57374a09b5e5368916176439", + "attachment_id": null, + "used": false, + "result": "not_actionable" + } + ], + "fresh_intent": [ + { + "key": "o/2a286f02c3854017886a0b371d08f2c9", + "attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6", + "used": false, + "result": "not_actionable", + "detail": "object present" + } + ], + "size_mismatch": [ + { + "key": "o/5be370c6443c51303e893c28529376b6", + "attachment_id": "a65a12d5-9369-4322-93ce-62f8f8a142a0", + "used": false, + "result": "applied" + } + ], + "checksum_mismatch": [ + { + "key": "o/77f4fbc630a1966cb3f6d4e09c019318", + "attachment_id": "ed0ff4c7-507b-44cd-8928-622ea6e977a8", + "used": false, + "result": "applied" + }, + { + "key": "o/ee7f3d21bebb4a56572b9b1f7d2299ee", + "attachment_id": "4c72dcfe-c46f-47a0-a442-2937c722ed99", + "used": false, + "result": "applied" + } + ], + "missing_object": [ + { + "key": "o/ac353be8ed77f5c230f7be71809ce405", + "attachment_id": "89e1b8ca-e554-4691-92b0-667b46757d10", + "used": true, + "result": "applied" + } + ], + "restored_object": [ + { + "key": "o/adc1d649f5b94260bd28445fb55ac5fa", + "attachment_id": "becdb821-076e-4bff-ada3-d0bf6823cce7", + "used": true, + "result": "applied" + } + ], + "orphan_object": [ + { + "key": "o/d2b0819448ab4d06a1dca06df4fac200", + "attachment_id": null, + "used": false, + "result": "applied" + } + ] + } +} \ No newline at end of file diff --git a/evidence/reconcile/faults-3-apply-again.json b/evidence/reconcile/faults-3-apply-again.json new file mode 100644 index 00000000..ca5ba041 --- /dev/null +++ b/evidence/reconcile/faults-3-apply-again.json @@ -0,0 +1,56 @@ +{ + "mode": "apply", + "grace_seconds": 30.0, + "verify_checksums": true, + "duration_seconds": 3.0, + "error": null, + "scanned": { + "objects": 187, + "rows": 187, + "consistent": 186 + }, + "findings": { + "orphan_object": 0, + "stale_intent": 0, + "missing_object": 0, + "size_mismatch": 0, + "checksum_mismatch": 0, + "restored_object": 0, + "fresh_orphan_object": 1, + "fresh_intent": 1, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 0, + "actions": { + "planned": 0, + "applied": 0, + "skipped": 0, + "failed": 0 + }, + "cost": { + "s3_list_requests": 1, + "s3_head_requests": 0, + "s3_get_bytes": 668556368, + "db_pages": 1, + "page_size": 1000 + }, + "samples": { + "fresh_orphan_object": [ + { + "key": "o/04232d5e57374a09b5e5368916176439", + "attachment_id": null, + "used": false, + "result": "not_actionable" + } + ], + "fresh_intent": [ + { + "key": "o/2a286f02c3854017886a0b371d08f2c9", + "attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6", + "used": false, + "result": "not_actionable", + "detail": "object present" + } + ] + } +} \ No newline at end of file diff --git a/evidence/reconcile/faults-4-after-restore.json b/evidence/reconcile/faults-4-after-restore.json new file mode 100644 index 00000000..66f604dc --- /dev/null +++ b/evidence/reconcile/faults-4-after-restore.json @@ -0,0 +1,64 @@ +{ + "mode": "apply", + "grace_seconds": 30.0, + "verify_checksums": true, + "duration_seconds": 3.548, + "error": null, + "scanned": { + "objects": 188, + "rows": 187, + "consistent": 185 + }, + "findings": { + "orphan_object": 1, + "stale_intent": 1, + "missing_object": 0, + "size_mismatch": 0, + "checksum_mismatch": 0, + "restored_object": 1, + "fresh_orphan_object": 0, + "fresh_intent": 0, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 1, + "actions": { + "planned": 3, + "applied": 3, + "skipped": 0, + "failed": 0 + }, + "cost": { + "s3_list_requests": 1, + "s3_head_requests": 2, + "s3_get_bytes": 668564124, + "db_pages": 1, + "page_size": 1000 + }, + "samples": { + "orphan_object": [ + { + "key": "o/04232d5e57374a09b5e5368916176439", + "attachment_id": null, + "used": false, + "result": "applied" + } + ], + "stale_intent": [ + { + "key": "o/2a286f02c3854017886a0b371d08f2c9", + "attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6", + "used": false, + "result": "applied", + "detail": "object present" + } + ], + "restored_object": [ + { + "key": "o/ac353be8ed77f5c230f7be71809ce405", + "attachment_id": "89e1b8ca-e554-4691-92b0-667b46757d10", + "used": true, + "result": "applied" + } + ] + } +} \ No newline at end of file diff --git a/evidence/reconcile/faults-summary.json b/evidence/reconcile/faults-summary.json new file mode 100644 index 00000000..d9b85fcd --- /dev/null +++ b/evidence/reconcile/faults-summary.json @@ -0,0 +1,226 @@ +{ + "date": "2026-10-25 15:11:30", + "commit": "f9f5ab9", + "grace_seconds": 30, + "scenario_a": { + "dry_run_changed_nothing": true, + "per_key": { + "dry-run": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": { + "kind": "missing_object", + "result": "planned", + "used": true + }, + "unused, object truncated D": { + "kind": "size_mismatch", + "result": "planned", + "used": false + }, + "quarantined but healthy E": { + "kind": "restored_object", + "result": "planned", + "used": true + }, + "same-size corruption F": { + "kind": "checksum_mismatch", + "result": "planned", + "used": false + }, + "old orphan object": { + "kind": "orphan_object", + "result": "planned", + "used": false + }, + "stale intent + object": { + "kind": "stale_intent", + "result": "planned", + "used": false + }, + "stale intent, no object": { + "kind": "stale_intent", + "result": "planned", + "used": false + }, + "fresh orphan (in-flight)": { + "kind": "fresh_orphan_object", + "result": "not_actionable", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "fresh_intent", + "result": "not_actionable", + "used": false + } + }, + "apply": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": { + "kind": "missing_object", + "result": "applied", + "used": true + }, + "unused, object truncated D": { + "kind": "size_mismatch", + "result": "applied", + "used": false + }, + "quarantined but healthy E": { + "kind": "restored_object", + "result": "applied", + "used": true + }, + "same-size corruption F": { + "kind": "checksum_mismatch", + "result": "applied", + "used": false + }, + "old orphan object": { + "kind": "orphan_object", + "result": "applied", + "used": false + }, + "stale intent + object": { + "kind": "stale_intent", + "result": "applied", + "used": false + }, + "stale intent, no object": { + "kind": "stale_intent", + "result": "applied", + "used": false + }, + "fresh orphan (in-flight)": { + "kind": "fresh_orphan_object", + "result": "not_actionable", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "fresh_intent", + "result": "not_actionable", + "used": false + } + }, + "apply again": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": null, + "unused, object truncated D": null, + "quarantined but healthy E": null, + "same-size corruption F": null, + "old orphan object": null, + "stale intent + object": null, + "stale intent, no object": null, + "fresh orphan (in-flight)": { + "kind": "fresh_orphan_object", + "result": "not_actionable", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "fresh_intent", + "result": "not_actionable", + "used": false + } + }, + "after restore of C": { + "healthy used A": null, + "healthy unused B": null, + "used, object deleted C": { + "kind": "restored_object", + "result": "applied", + "used": true + }, + "unused, object truncated D": null, + "quarantined but healthy E": null, + "same-size corruption F": null, + "old orphan object": null, + "stale intent + object": null, + "stale intent, no object": null, + "fresh orphan (in-flight)": { + "kind": "orphan_object", + "result": "applied", + "used": false + }, + "fresh intent (in-flight)": { + "kind": "stale_intent", + "result": "applied", + "used": false + } + } + }, + "objects_present_at_end (fresh items were older than --grace by the later runs)": { + "healthy used A": true, + "healthy unused B": true, + "used, object deleted C": true, + "unused, object truncated D": true, + "quarantined but healthy E": true, + "same-size corruption F": true, + "old orphan object": false, + "stale intent + object": false, + "stale intent, no object": false, + "fresh orphan (in-flight)": false, + "fresh intent (in-flight)": false + }, + "rows_after": { + "C": { + "status": "ready", + "lost_reason": null + }, + "D": { + "status": "ready", + "lost_reason": "size_mismatch" + }, + "E": { + "status": "ready", + "lost_reason": null + }, + "F": { + "status": "ready", + "lost_reason": "checksum_mismatch" + }, + "A": { + "status": "ready", + "lost_reason": null + }, + "B": { + "status": "ready", + "lost_reason": null + } + }, + "content_status_after": { + "A": 200, + "B": 200, + "C": 200, + "D": 404, + "E": 200, + "F": 404 + }, + "durations_s": [ + 3.94, + 3.983, + 3.0, + 3.548 + ] + }, + "scenario_b_interrupted": { + "orphans_injected": 2000, + "interrupted_run": { + "exit_code": 1, + "error": "store_unavailable: Unavailable", + "applied": 440 + }, + "orphans_left_after_interrupt": 1560, + "rerun": { + "exit_code": 0, + "applied": 1562, + "error": null + }, + "third": { + "exit_code": 0, + "planned": 0 + }, + "orphans_left_at_end": 0 + } +} \ No newline at end of file diff --git a/evidence/reconcile/interrupted-1-apply.json b/evidence/reconcile/interrupted-1-apply.json new file mode 100644 index 00000000..6f9ab9c7 --- /dev/null +++ b/evidence/reconcile/interrupted-1-apply.json @@ -0,0 +1,6055 @@ +{ + "mode": "apply", + "grace_seconds": 30.0, + "verify_checksums": false, + "duration_seconds": 8.422, + "error": "store_unavailable: Unavailable", + "scanned": { + "objects": 2186, + "rows": 186, + "consistent": 184 + }, + "findings": { + "orphan_object": 2000, + "stale_intent": 0, + "missing_object": 0, + "size_mismatch": 0, + "checksum_mismatch": 0, + "restored_object": 2, + "fresh_orphan_object": 0, + "fresh_intent": 0, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 0, + "actions": { + "planned": 2002, + "applied": 440, + "skipped": 0, + "failed": 1 + }, + "cost": { + "s3_list_requests": 3, + "s3_head_requests": 440, + "s3_get_bytes": 0, + "db_pages": 1, + "page_size": 1000 + }, + "samples": { + "orphan_object": [ + { + "key": "o/00213cfd45224daa9efcca3a4f659652", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/00709b4df6e7467f822701e12ddef6c4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/007f295f26ce451aacb50f8917df5df5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/00a1eef51e8a4f06b9cf4483a39f0527", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/00a7d737898941cf997002e14a83b4ae", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/00b8bd484efa4779a85c77297a550fa8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/00ccf54dc9724782b6598aa2c0d31816", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/00e777a9cb3e4431bd8bd09fbd1586c9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0109a2a589164dc98091a7fda02ba189", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/010ebf95b3e04430b45029f143864741", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01228ea1f561485ca7fef752437fec26", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0126ea6c44354774aa0339a6e53763ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01386157f4f14c3b8c045b8386aa5bc6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0169c1e75c084dc8ade2cc6eb60af987", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01aeede04367464eaef0e47ec0fe312b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01b062136a7f4f5c955e95027b77ed9a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01b8c50a6a1e4885a77f69400478cccc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01d835c92b584935bcabc81041108841", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01dd8a1b07214e0fa65196c0b4ede7bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/01e2524c1e0546ffb5bb3689255e4adb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/025fced77bdf456a8d46ec6b26c4767e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/02bf8677a7144593bc4281840fd37371", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/02c2594b34cb4cce84c97e9db7e44337", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/02f91b045df94027bce4c61d4009167c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/02fdacc22fa548fca114b39a3bddcdb0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/030b5a23b6434c459d776470386b32e5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0312b0a49f4d4f26ab7b23980682c898", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0314e22ce0a549efa9fc8f1f6346f6f1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/03375fd2173a402bb52352bcf98bfd1c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/034215cdf3084113880f5d29ee3944a2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/036bbdd20b704983bb4770b2f37a85a1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/039fef9026fc410c8bdbaa2925ed7de7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/03a30f76b1124c16a6b496d29e14dcbe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/03adbbd0b0c64cdebc8b27b9fefab111", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/03e23db193694feb9fcc5be87f4f01df", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/041d584714254b0c93f9570965939b00", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/047e8f747bb24e4f92dd69c49fbb55e6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/048c7d54bfe94ca3ba78ce7bf59c960a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/04a4c96a03cf436e9b9208fd82f12bd0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/04a61de5e69d41b48adc0ea348fe0998", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/04af3702709a4222bba1d3dc0cdaaf8c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/04d2d034591e446696a63ac15965e859", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0525036e42b74312911db0a00f09b695", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/059d5163b37d4daf8c6f374f25a23ea1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/059d8a25842b443abc0d040b3c256765", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/05ac9ae53265497cbf8404df665a1f73", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/05e71db8120949569e72c6424dc8e1da", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/05e8f539f59b492aa2c405a7b48aeeb8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/05f978de6f3a461797d970bf0dffffd0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0624eaa212f747a4a541bd1878d45212", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/06ac20c2e8ac4907ba648ff149062724", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/06e89842336642e68c14808fe4873cc9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0712f3ca08a5441b81958df902870ba1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/071e6fd6040b4f4c87d30850a629ca59", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0751b10e5a7d448b9dc935eb6e417fca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0760197b030a425592dec0d2c3958902", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/07a5cb771e8f4999a652f7a65e18a135", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/082426deafc24f5f9c78ab4faf118879", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/085641d130a04b5cbe834c147ea6d184", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/086c0a61964c474a942eb2ed73b9e61b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/088ac5d55c804f7cae8d99d105849b06", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0894c974b9334b42aaeadb0b5726f4be", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/08973a9f47ca4ede87ca20de1c331fb9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/08e38a9df0eb40519b4edb96ec596855", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/091bfd7517c54c59ad1befd16a85ce72", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/094458f0ec9746399f6c9f274d81bcaa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0947fee656b54431b18d482172151fd1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0952af1819e5496f89a1cc064d910088", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/09846cc5ad754967bec9074545a78c1e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/099951b614f946d8a957123397b1b197", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/09acdc6f4865415eab37b47005bd28b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/09b8977e7f7043f5af1914e0aa5adae3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0a4a5ae5621545798782f1c8c66befc1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0a4e927fac4d40eeb1a339b6d917883a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0a5e58b26c864606b4f0f20e4baf5383", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0a6004d94ee74a8e8b281a9faf20fac3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0a60c59ee1a34e7199c08bf96ac98446", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0a8893ec498b4b50ab6c2cc279f48b23", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0aca8cdfdd91496b87554b06ea391a20", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0ae15202c28c4eb8bf45c7a2e9ee90dc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0b04460aac9544dcbb237ed0c441be43", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0b09cc5dd94643f6b78c30909d05bdf5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0b30d3ac888b4253afecea66b7ed0105", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0b88d9794886427ab5bbb29ddc8c0bea", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0b89f94022b0416a9f5a46b287e9161a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0baaec85f81a453ba048186d7c1c5503", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0bb2f1ad730c49aab76b415ffe46ddc7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0bb6bb6fb8694058be3056c653309956", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0bdb92fcfedd4f78bae546678ab2c1a1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0bdf8a0706f64b038f25b3197082cb41", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0be9f1ccdb5c4f89b5d67629e5d206f9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0bf5fe92dcdd453bba9b44dafbe8bc61", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0bfed9f286504bcdb080afeeecea4c49", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0c1032033a604ebbae8871efedced22e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0c2603e933f040bd92fd29c6eaf69a3d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0c4593a4729e40209acbc28c5b0b5235", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0c648034cdfd44339628c678df484ee4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0caac8970caa4b5980373d031e909445", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0cc20e62aeaa468eb2e99c40c75ca3b6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0cc93633f60d46c89059903cb3372de2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d18370033774f9d9dcc8e0a02cfc3ab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d1b06192e44468288ae9cd5152b3a05", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d20288e7d674a2c9151c7de0a519d04", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d232be8cf284ea99e46642d8d8316ef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d5005091e5644f7b7c8998862c55573", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d6d01a1d5f444e196dc89d196588c69", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0d96bcad83d64ddb94744e85c1e1067e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0df0138996474ba89757d658e4746d64", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0e00cf72e8d04c0b9adc246f574b0a39", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0e05b3b654d34dfb908d350320f8fdd2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0e06c646bfc546b09e4a6cac9fbf34c4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0e0ccaa13dd34ea4a703c756da1b757e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0e1fc2a9aa634bcb9c653ecbfd2844b1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0e9fac176916476fb79a180aeb67b74f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0ea0f7203a894ea5b516babb771eca42", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0ed696387213486786c0103404cccfa0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0ed9b88679354b3299504fdc644f4add", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0ee93b76ecc44c19a8a14c2614c14ba2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0efc855670ee4e06a2d23b25b89d3663", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0f113d21a02748c9954effe8d8f1729a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0f43314618494484a414248b00f225be", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0f5374cb68ea4dc1b456c7a313378edc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0f5f7c774d7b4d98ba243264a01dc007", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0f760ad47ba84409909d8048474d50d9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0f7d3dd0a15145aaa185723150912ade", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0fb96d46e87a4351b38130633d80990f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0fd41643b5714536ac84da9136536fe5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/0fe7136414b84591a316be63769f07da", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/10078cd26a4243f6b253ade6c58112e0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1008c38ce31e4a2d987a4555b4b3d9c4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1021d8bc32f84ac387faa26b6fe73f9e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/10222a7732834543a4ba0f9bad6ef033", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1026e488c9bc418281ebf47ffc0f7ffb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/10417483feb24321a79d7d0364d78f2f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1061ca4d10694f3c8976a71ffdf7ed79", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/108bd074581840db870be407a8c96d95", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1139bff5ed264952959a6261b8d87c2d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/115a8681017e491db9b0c187f6403ee1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1173321dbf9742ecb5b63d2fcf9036fb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1182d4d65ea64320af18b0425e9f50bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1199bf75ea724b269d238eb2f4f10d75", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/11d21c59ce954a2ea58c4ca0218f48b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/11d2fa9f33ad4f89b3255612e6fa1d8d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1203014b13784bfcafde324c43b7925d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1221bd887aec4e7091dbc5f0a48bb156", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1225336343de46478e45fa4d5dd97d93", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/124665d319864e3ab77a5a91f5e5962b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/12bfe90f3560491ea97797043a5453c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/12ff5c650eda47cfb3c0fa6cd2755200", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/13119b6a513e458693ee1f4c3e0fc6ea", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/13189adf371a4276871cb8dc2f6e2197", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/13506f53367f4938899ae6e57e761706", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1394d895b968487b9112a57cbd8b2218", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/13e7940f8ca341abbd7ec28093bc7c87", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/140a2ce8e56443d5a7c2e87f0fa13133", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/142e3e483e5a4e0e916a305c71cd1501", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/144122439e004b4989806e1e813c286b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/14f234c482994016ba0f4f68b993e636", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/15256229363641129c33d2bd3291b703", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1577eee5126049b79bae447db043382f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/159e124ead66405c9122d7f1374fd27d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16091c694fc243778b00a4aa62da7fc1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16141d5d9d664beab52ed3735d089df1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16427e9b02bb46bd8ad504bee3700d6b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16439fc0e0a8401cb49682ed99e5db01", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16623b174bb44d5888a03eb8a82d5dae", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1673ac6fb9404c95bd4f55118edb79d3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/168451728551487b984c29972680aa03", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16e1c93e62e44a85ae731b92da23eeac", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16e266c4474a4621872a365c082d2a7d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/16eb00ed03854665ba110a4cc11f2ac8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/170b07b11d864fd5b490a373dae7800c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/178b5dadb8af4e58958662eddf519f68", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/18144923e66b45a9a3448816de656073", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/181d71c34f25425d911f1673acf35731", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/18896a958afe46e4b440417d75d0a4b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/18d23af3a8e0467da96ab85dc15175e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1907105580bf4c48b2e2d8b1c4ca6541", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/190d913d332d42a18f0daf532dc66419", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/193333f45bfd4957b78dd393c2e74900", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/194e159bd884446ea96f6ced9cce7e0b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/198e6b0137c140b2a91ef2612f68bbcf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/19c171c784a544218f54f50d9c6e9b32", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1a230899c1c94e86b9e1e14a03b0dfdf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ad18e62a3a949eeb1b83d0c1ff56f77", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1b8e5ad32f0e4301ab12e3f7ca9d27b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1bc00dd138c046048dc9d6ec61031be2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1c1d45f5ddaf4f939571772ec43a1ea9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1c1ffcc8650043569dcfb3cdc0a42116", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1c368e0a875c4ef48d636f577048a014", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1c3edeaa371941dbacf7cd0e290b39c9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1c41a0028fb24cb3be1d029db13fa1f0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1c4cf8986ef74c8eb5b2b96f766defb4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ca9046838a9404b981e97c424fd0277", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1cba9e5c234046e0806986deedc56851", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1cbfff13acae42fab5d04b3be33b163e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1d050216de6c4115b432c1ee6fd20853", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1d49b5ebd0cd4318b388be5c39bb478d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1d78e5a5c325400b8077ebb64d209bbb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1dbbc7667c2c457b92cf8784c6d66af3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1e09e74fd99f40da82aa061891fdd1ee", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1e4b2ddd2a404502b719dbd2375c7f18", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1e91bf1660c44110bfdc762e99963e11", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1eaacac8cf08463b970fc372e291d69f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ebc0a5396814a47a9dec0698824a83c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ec26d0a44124d3ea8325bc6667085e4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ed11c7e6dbf407cab4c289d64bbec43", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ed8f098926f4b7eb20785fea6664458", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ef68d60eee649eea876b8293f2b55ca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1f0051e13ba5446b8eb2f22a398bbffb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1f147d15432e42d4bbbff52943794366", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1f50e780434545d79b541ab906014553", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1f5fd757a26f460f87eeb930dfbc4af0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1f7280a0b5e24321b6dfa027c1328df4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1f8e9436ef884fbca3912d1e815d0501", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1fcb46e87c814246bfd1c5fde1ce6110", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1fecc36ba2cb44baac69d1de3bb535d3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/1ff65eaa2a11432182cc62cca11efab2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/202c1419d0fe407883a888d564b35abc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/20309dc7222747beab3b0eccceb3d8db", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2045debc4ad547c7b1c9b6dc6b3dfb85", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2059bda709bc4366a9920e1b6fe75301", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/207f351b46b946e0be10ef05951767b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2089b15c6ca643199aaed3aaf68df293", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/20a1876611b84c96ad7dd7cf5da2fd05", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/20f9309bb17f4a89937ad6fc180c3915", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2122f68ddf3a469588c6304d8733b442", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/21289356a368435bb13e1db0539e2698", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/212c6049f9444c6aaf8db8c774858340", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/212fbbd84c7d4ebba4c9a1c865021ee4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2135785030d14ad887621997edd27ffb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/21477882f5654307ac789879eecf8766", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/217bb85da1234e07b14b00b17e053b36", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2183fca05d5d4088a6a0a1bf900224aa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/21982aab71694fb0a635dc9479697810", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/21d5afbb44b44b84aec729c34e4aa9fc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/21e8d94dc03c406494fcb4d5a82f2ba0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/22064a9a39ec487499cbd66247f1bef6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2238c40345c141b5b55977cf4f1acd48", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/226c80f3d24a430688e3701cf53c79a2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/227b2635e7ef437e81b7f5d93d652b1d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/229ac4126df2424fa5d14e2797d4898f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/229f0f60a4b041539a5cfe68df9f036f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/22eff882d8da48d9bec3853f1ead2573", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2363a52013dd4d508e7222221f718cb2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/23f7049770264e9f880d905540a49a97", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24071ecdc83d4857947d9940fc4ae279", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/241143bf87bb4c6694f25b652a5ce5b7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2458f2a5177b4934bba6e36d47196c75", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2468bb5d4c5645d1b33432740d6f139f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/247d68fd879a49618df1c33d0a1ad076", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/247f512b53ed4cc894def595e7e645a4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2482b3af66f346dcbb6f6754f36c12fe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2486cdf6ddd94813ae8578a0a86fa461", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/249d1a0aded548609b16c8007963c06a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24aa6f93b96544108012a6e47a21a779", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24b50c6f01bf4ec691d7044c9f7b55e6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24b8edb2d4f24faaa4b6d7e8a2978c14", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24ccadd7a3f6487491d1e3c42f407cef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24d224460df546bcb3181b4671ae07d1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/24e8181a3d1449c8bfef5c5fd343e7a7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/254e4f51db5f4fa68777ee3bc46a4575", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/255e556b7ee64fd09f473b8145da45fe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2570d758a53346b681ef97b60591a1f7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/257855a157c34b5b9b60976f9619565f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/25c42fd62aaa4931acabedc14605ecab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/25f53dbf854144d8a39c669d8c1e0d3c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/264a4b3f85924b1d88b3c2a87f69bffb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/267092fce7264df99ce8631fb38fb2f9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/267ee1824b05472d88901052b2b1f314", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/26937fd21b2f4d13ad5c7a841a2e341e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/26bf9978ada24d2fafe3abe3978f90e0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/26c51867be864ed793d567123c201fa1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/271911f7e8fa45a89d16a9d27f599e12", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/277aba8f15cd41b68a55e22334a10f9f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2786d3fcb61d4f64b74f3dce10456223", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/27ad57ce37f143edab6699c1346fa083", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/27ce1315717942a6a571aa988a2b6715", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2813be6808b94459b445d33eeb0d02b0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2868e696c6b542488abe5ab1da61f52b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/28f4b437c36a4dd6a28ad1b17dadc03a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/293e6a12c54749628e231a9b4340d06c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/294e99c9d193401e9c50ccf770e22d88", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/294efdaa4a054ba6bf1fbee262dfdf31", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/296dc8d6918a4266a1fe1c4b263c99f8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2974db86131e4e43a96a5dc4efb59746", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/299dd23adb3f47d1a327d1847e810141", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/29a9c399b7cf4c13972761562da5d75a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/29da759a200f4804953702d6c5f1d5bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/29fc10b2b01c428a9e9990edce993785", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/29fe5e399d2e4686bffdc4a100aa6bda", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2a32020341f842fd88366be103dc3616", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2a63feab62414092bad13822febe62d5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2a79651f6ac642f491921f62a64cdaee", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ace20d26a8d4fdd9f18c15c861d5497", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ae641fc82c94618a5e969ce89cca637", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2b3a1b0c25ad458a98b1e7d32ce6facc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2b41d8cdcfce46f498a3ea1ea74d93a9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2b77495361eb4d5abe08d076f36943d8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2bac4d173a7e4ceca61962447c0e130c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2bae14b06d4645789dcf313936c6d4b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2bbbb083a09f48aba6e3606400f309a5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2bc4e8fac5bd4deb8026b083d20b48a4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2bcd195775e7447fbaaaecc796363345", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2bd9d6f90d644953a3b94832d846dc1b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2c116ff15f83411f9b28b3952d0f3737", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2c494989f8d049f8ad829a625b9c33b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2c4eb4f88d72467993ab86bc26260b32", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2cbad13e11a243cb9a53c50f3115a131", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2cd3f0d659364c3089d8dc42214ef860", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ce4beef20c647798fd0e97c2e787de4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d2195b86d5148c7b6327b9fe2bd31eb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d310bdbbfe447bbb8ff4b97570d5169", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d337d4ee5184afeba13aa00efcfc262", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d4f542e20db41d2b91dee04147ae6ab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d73ef2a9677468698b6053925475735", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d769c4bc0c446fbbcd394cf84aa7448", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2d81c95913ad4c4a9275fea1f6f13595", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2da357f691aa440fb6c4fcc936d46778", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2dcc7b2b82c14fde8dbec33c15511060", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2e1a724cd09345f994f6bd8e8fb581bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2e4464b8b9124511970ea7f1d5978b5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2e4b93a34c364532a0f0b18c80295d56", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2e562d8c49be4496a6cf343ad1d36416", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2e83bab19f2f4954bf77a8cd36dcbff5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2eb2fed1fdbf47d7a2e382ec3027ce65", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ebc513385d24bc0b6f3aacd6efdc4a8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ebfcdb078ba4f07b12f34a03eeacf0e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ed7260e2e2d4bfcb542604f49f3e1e5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ef2600992be40bb93b064305b2c3edf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ef922187777491f8eeaf0d4a49179af", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2effaf8c1af04523acfe44f8fd9bd959", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2f1f2535a1a44e5f8c2b9e2515f25e58", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2f6c4f1978fe43c189e73a1b9243609d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2f7222cc71564ffd884c4ff97a84ffbe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2f859faaf66c499a8ffdc4b5db457832", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2f9dad0dfd5749b6acbcb4f593c72f58", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ff6b7605fa145bd9adc964f314a14c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/2ffca72bcb624e47a79800ada91b01e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3007b5138ffb4262b0450b7ea67e5f78", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/300e25cac64e4c4982fe3e233f9e6916", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3019f233ea5a4544aef31eb575223f32", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/301afefaccde483f80eeb311dcfd50e3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/303334ed95af42a1af767184d0020564", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/30872fa49d2049d6834cab82ef50ed0f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3095f580656d4c64be323ca121573fa9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3112a9e06d3f4c59b31b5e66fe002395", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/311feea16d9b42babf9a7093077fda14", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/314498eb559544c8a17bf4ee9834de67", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/314c536a19514b3bb3f24d7e090eab9a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3153e1412c644c88be6ffa428e722617", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3162653c3f2740699ddb348294f1ea49", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/316720b63544490592b46eecf601f1ea", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3180aad9c7e8476bbcaf085e87948353", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/31b79989c7964c7fb6d1c7b6d783dbf0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/322280a8621f4dd393e14eaab1b9120c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3233fb4b17c14f58bfbd456cdaa4d615", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/32ff66614c864c13bbf925aa6310c774", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/337d83bc957840eca031fac51df0136f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3392e316d50d4bdc98e0e54d927141b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/33f0c51d26884ba6b3d0028b7166a65d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/33f799aed7e346f787ce1656b9751584", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/341c8303ed2441c9b848caa51c4bf20a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3422ca0d03e94cd4b02dfa4e2294a669", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/34b6a05b372344c08322ae47662b1e43", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/34dfe17008b04909b5a9abf13630c7f6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/34e8f5e83e4c4f719e8a822e846c9b7f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/34ec128719bf4744bb53a26da86e47c6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/35185cde1c93428b89dc26ad13efb7ee", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/352a97ee2b5842688143fc6b4eaf076d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/358a7b9651f84f208eac05642aa1ae8b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/35bc0260710f457fa3eab80dec4df993", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/35f67120c9a847d4874ac10abc6fbaa2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/35f6f204848043d69ac01bcd8dfe332e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3615e9f58aee4bcdb7618583e0fc2d5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3636a7abb25149749cc9853eedf56537", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3649636efa4e4acfacfd2c5df27da632", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3694c1427ef748c28b4b2f3105129af5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/36c0d280b45b4314b830bed759d80b77", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/36d8f17e48c54cd6a3bf8d98eba7dd9d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3770f0024ae14d6cae1125d8e0763a01", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3774ee2fd17f4100b76129e39c550d49", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/37896f0f5ee04ba7b721d638b7566f59", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/37a6b5efd59748ac9bed346458eaef0f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/37ac3a72734b4669be724d599ab1a2c0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/37afe5e291a74b0abd683b4417861894", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/37be9e476cb941e6a5b5c4ecb13aba5a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/37de95057b8a4f3e9fde8fd6817f2a49", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/380493b472994d1591e842eb490a33b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3816a745453e41aeb362a0165351a4c0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3825e06a58a94228b7362343611c71ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3829eb6a94a6443aa7091e52143f1826", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/38b6ff9667a241fda88ebf7b372a1b64", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/390a26a8c3de44ff81d8ce94d88aaf42", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/390d9f9cba9b4e98ad6986b05f54830b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/39227879623b4d2a9eddb782b246a465", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/398cd5ccacc44954939fc4416db59fdf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/39fc32db74464908bf41606e88063c52", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3a2fe5e8b6ac426689333b3a57655fc8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3a8bc6ec1a85469489db190fbd3a464d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3abfa9bbd43b40658b42bd8ec8c55df4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3acd137eb9c94a3a93b97e2a9fe56879", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3aea97f21f1f42fbb87c8f80d2d34ff7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3aff8c008b5c4d8288e09a139942c9c2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3b6454345c1646828b9cdec2088ccf04", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3b6b9b32558a48bba58973e00a2bb378", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3b935523e0674625a867fa79d9ca1df5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3bbcfb22f6df446e929e149f659e3b68", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3bd4df954f9c407d8922251ea3e51299", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3bda6cb1f8fe4485b956aa955e5b40ba", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3bdbf2a1b533419aa6048eb8fd39195d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3bdf3d6304b94e608df66eb22c8eac91", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3c304753324942aa82bf1d236ccefcda", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3c63e4e8ddbc4fa1904f5ca9dc8be11b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3c65cc74959e4143b0fa223bf9da0c76", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3c6a85c2d2734ff6b17376fa21763072", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3c9f2622dce042d9977b5086b1afae18", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3ca91bca72d94c9d85de927ef1cbbc21", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3d304b5293434755add7b0b7596447eb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3d39b993c7f9403f9ce566bb6e85016a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3d6df03ce82a4cb89237e3219d522613", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3d8675cd1c7d4d378bf3982f6a76dd0d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3d9320ca630e4bdf9d200e828466e3c1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3da7c193a4da4c39949c5a1e0303d4b5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3dc1e15df21247479e13089d2b56bca5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3dd97e4c277e4fba811418e163dc2ebe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3ddab7f78cfb4f63ba11e3185d91e8ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3e332bba037b4bf19b2d616f720b5127", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3e4869520463419bb6305e38f179ce4b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3e65fdb658f84a9a9d48bb5cd5f6e22f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3e6d6d0a7d7c450fa0b382753bf8ab2c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3ea2ceb8cc6e4db1a5d14202dfa60d81", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3ee7b40f31214619b610fe46c938c9ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3f054414f17f473e870562cfd9da38db", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3f07cc70c7264d92bccd886578f96e31", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3f34ee29bf5c4958877a6309f0f90fff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3f3b680bf2a049538f8b076812b9c3b7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3f48643877204aafbeb3dfbcdf1f1860", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3f7a2a31700241cab9ed9fbd974f3593", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3fc908e0061645659fe96ba3dcfbbdcf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/3fcfc7880b8f4924b8e1a16164af8980", + "attachment_id": null, + "used": false, + "result": "failed:Unavailable" + }, + { + "key": "o/406dcde1f0ff46b7b15139434125cb89", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/40b43f8a2bf04b68b377c1e0a6ecf525", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/40e497e353cd4215af041946a0c26b9b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4105740884cd4215b0f66c4d8f4ab681", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/411289b7102a4e76a6dcce99c280cb27", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/41357c3da64845f481af9c1ef299ea9e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/414e6e3ed1da435085230fede8a3be95", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/41a5bff616aa42c689e2ee0753d036bc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/41f49bb8f0314f5ab8e3b33f1cb98066", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/420b27329f6f4162ba1fa98bcb1396b5", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/42238335bb9f43a6ab94d1a0587f4919", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4224f05c869446c7a2425394471843f0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/423d034b5693478c9c72d0f9cc3a313f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/42597cde33fc40799c5a94a3da77b2d3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4266e0c5c4aa4635971802d33e4e6242", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/428a288d12a74f6ba33371be745c1533", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/42a3e929a72f40f3ad4e0121ca5f01d5", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/42a961603d544c3ca214701870219655", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/43120578859d42f6974e0274c7188b95", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/431d3d3db964453aad41ff0ec3babed7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4344d5e732d74ad8a1bb5ffcfd135b62", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/437fba8edba64c6dbb4e93a34f90fd0e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/43862bc845644860aa847b2a584fcb7e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/438f9d52a0a84d4db2533dad88e0605f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4392b7187bca48dc913fde23989c9e4b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/43b8db793f3545da9fb5fa38e2d4ec43", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/43eca42668684e54aafe37e77051e2ff", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/441535cceb244018997ed9d70c7054df", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/44310108170443bebcef3a8aea90c606", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/443aa2c6f6fd4c06a34ddf66a0ea3200", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/444a98dfa5d5494d8007dc892e80f296", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/44581d4ffc104eb6a0bcfe1cb364fbef", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4466f206dcd84d0f84a699a5cc5ca90d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/44826520ad1b4adba667336a92f22889", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4494042a5ce84e69bf48de1f83689300", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/44988e38804042dbbef3a1f38042cd93", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/451160b197b242d091bb42cdfd1722e2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4512223d967643608842959696f7a5d8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/453201b972534a17a3731d27b33a6eaa", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/453d3b0b07ad4102b8fe9244710260b0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4563874dad62475f8cf36e707bcfda29", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/456f527e2d5b4eaeb4a7fbe813dc9f0a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/45c32b716181481f9df377a22f3579b4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/45d4bd587d6e4865932cb9f1780fff0c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/45d4d80e9e67473ebe633fde6f74ea2a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/469af1343b2448c392126f420f0734c7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/46af780ad90348c6a44a1849ba3b5045", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/46b4a8edd3fe4c33a82e5acfcd3161fa", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/46dd9e4430144d8aa183a3e7e377ac7e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/46e8b7b5bd184e1fba285fcfe49b21a7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/46eea46534f94489a1ca97a4cc58ab50", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/46fad16980e54d30842cf0b7c53be4ff", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4705f8b4399842a5b2e3a3237b0d8f94", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/47064db5672047d191aa348e20169b31", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/47356bb6ddfe438a8837119329c7073b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4737ed9e731f4c4ba200c064e1114013", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/476e49019f76450f81fe6826381fede2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4776386343b347628f1499281b5a0fbf", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/477a11a3c5e34e54a85099b91f4a9bb6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/477c392e4aa54f999399fe867bdadd94", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/477d5d557d484cdf9ec9f2f7d348e17b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/47809e39f893440391439cf50c4984ed", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4784de5611c84e1cb5c031df06cc206c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/47ac6af4d1304fbda0ba9423f861cb88", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/47bb74c16a4c42ec9d45eaf71f0a1d67", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/47d9b07a8a5747e99d53f8cac029bd89", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4832d4af593e44faa8a8a38acc2e555a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/48a5cd7b296c406f895db5eb5c44cfc9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/48ba443e37be4bddad5c174d5020982e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/48ea081b62314d9496e1f9043ec37bbc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/495ddab1778b4217aac9a5dcd6857285", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/497c3ac93bfc4b7e8e369c89240019f0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4992a3e1eae547dd8646afa1424ded14", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/49af3ac368be42a4900eebb3b1d2c6c7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/49dfdf4d96654021914301166e65ff73", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/49f3693bdcac43f1b50aabdff556a6de", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4a1f01aa66c1493f80cca5dc7f09e001", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4a6435baf3ac4d78a3234e16897319da", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4ab4f021a89b4ac698935be7d035e43f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4ad69eb583784a37a6874dbae0a1a8d0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4adb920affbc437fa0e09b3f28557788", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4aff7b79e80d4b38894e1f6f609b7185", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4b195cdfd9ac47119b228065022e5c14", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4b214e27c9f2497db89a0a186b1913ed", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4b3948a665694e3186ecab562221213d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4b63717ab4754837ba17b887085ccd46", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4bd71ab669a9416c8efeb78c782d3648", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4bf9a3ccee2c42e4b4ca81cda3e3b54c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4c107266825045d2976d9a7a3a5dc99a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4c14b5683b8b4b20aab0571410f4283a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4c442b49d7634d918eb3864a2f6bb077", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4c7ef97aca9940bb997289303e911827", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4ca451c259664da8aa6324a1fb5ac495", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4cba8495b7a545d2a9d2c556b05a4f1e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4cc66ff9380d456cb332b4181f165818", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4cea96a4f7e1439a8850d539f10dc2c4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4cffe16bc06c46a5904e9117b2298da2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4d16b2ca229f4f9cbf2ce03c08be02f3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4d19768d78eb4b9b998cacc95843f274", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4d5c8e4d17eb4ebe93b7c7199c9b6029", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4d5df3d192a84c02b93ebbee86732255", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4d7d0059c3ae430db1b43c65fe511242", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4d7f812d745e4ab38eb2f41095df3b0b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4db91e89937d42eeabcbb678ac67ff8a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4dba51d2bc3e48d999f39369f7efd02f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4de8ff0d033748a0b05cd1bbcf9a6616", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4df8857f0644447981c844c57a1210c6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4dfb636e2de74f2d873c776eb218ebf3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e060f3b6b1d45f9a4905c86963f66fa", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e0bb1ed5ec14637bb106b3a159bdc37", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e0e52c36d2442099a1662c8e0e876d4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e2a837216154f40b025bc1869fffa11", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e79a33ccf2a4c04be4386ce902ff4bf", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e7f820d19ca4b2180504679d610a5bc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4e8b6f2f0b9a4d7d8bb1f39e4a2a045a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4f0a2d3c515f41d28ad519c594b590ba", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4fa94bdfbe7e4ff1abb6b7b6adc4bdef", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/4fc7f33a0f8b43f0b9aa35c87545628a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/50425f113d1d4551a9bc5096df572ddd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5055979b15014dc098a9ad8c7f187bd6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/505b298298b540419d7bf408ba92e317", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/50b3966f373a4767942fb2762250467c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/50e80ec2ef4b435883331049daba3dda", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/50fa7bd8fd164bfd918d6cc2b4128ff3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5106c60032be46679aa4c84582c447e8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5119ed50d87142648f40af6d270dee6d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/514e5d12d6594d258bf85a6f3d01ee99", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/51ebe5b069b14d71b09b1dc086ae5edb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/520a5a489f5049089cc9b64d95726c03", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/520eb1db7bd948c3a5b0907034563a39", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5220b75d311541e0adf6e6b39b0b8aa7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/525f4237bf634b8b8c3b3a531c206194", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52678e8f2b474c98b788be0a892be815", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/529d8d7b43e641d29cada8d2fa08d17a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52a437c3035c4b23bb1a946048e2c7ff", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52abf0c22a1a4bbfb7cd6e1cbeed6a38", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52cca875da374e0a9efafe3061c9cabb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52df06c3f08b482cae786ca09c937ff7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52e5cd5af3884254907857babcd48677", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/52ead265e5e54fcb9edaddb28bf70618", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/53037f957b2e41f0ac1c5408fa554b6f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/53538d063a114b0395943612ecc8e710", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5387b490ade1496fbe814b13e271072b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/538933eb554843d0af04363ab8adfb81", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5398c9e1cd3d420e933dfd9911127279", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/539d4a8797264a9d8d999f5b2ce4ee9f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/54071bb6fe6048a086107c56f6d3d50a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/541648589d224a7bb76976f91a4c0052", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/546ce56bc32f4854bad49c93bfa1a803", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/546f8c8c7a7b4962a95ef2d268b17d38", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/54a815a4ca444a0f924e3d0bac58efad", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/54b2144f0cf2485cbaf824583c98fe24", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/54f616cf3615413da2ee9936b87f9da5", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/54f688e7ca77476a9394489878f31188", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/550e006e700e4d8ca73fb4f46bf0ad36", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5512a26bebfd433ea95c4eea01524655", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5512a74cdd484e40b4c9abd6e316797d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5524b4d455b24c93b6ad3a6899f420b3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/553f539a2d7d4068b02485ba0d89a603", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/55626174b0fe4263abaeae54e3450b45", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/55687f2648f645d587153f009d1c4158", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/55708ba805954202a36d5ee0282a0cba", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/557bfe6b61d34392b29569491c51faf7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/558fe4d4d6fb448097a8849cd3f638d4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/55e8d2baad8645898e751d5eae762cf3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/55f7465e1666447389b7c513cd1739bc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/56139b94586d4839b904bf70b5744d2d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/564756548d2b4d6082129155919b241b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/56cb1b27997343a1be03e3fdc4284bf6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/56ec0a8ea721453ebca2cbe10ffbaf0f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/56f3c3d406374c70a53931ae34751c52", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/56f457afb8974ffaab760bdee663a90a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/57182ae676ba4d5090184397f063d2fd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/57394346e0cb48358513718b2e0a3288", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5749abe433824c2ca8116e2c796e96df", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/57a4377354844c91bb110b5f336422de", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/57ac031447a04e9f87c8c7c451a135f8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/57b566bec95541e1ad1509299592e942", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/57ba5a92424d445d9b7370b817509289", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/580b68885eed498ea79cb910b9326aa4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/58177778ae7446bb9a862ed8d08afbec", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5836b13b18504596a670be13f239c9fd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/585ee17ea6c44f5d842fb29abd3153d7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/58f5b284bf61461ca15ee288b64130d3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/590bbe0c2d3b473faa83503e0aa6cd06", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5936b6a7f9c04056a14e38dcf3346b98", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/594b7690533f40d285b33d889869cdd4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5970a038263c4d3f91239085efbf1558", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5972e161c2c1464b95d5016580f50ea4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/599c7604efc542d98311258fc02e87cc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59a5b1dfd7e1426280ee362bfe1859cf", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59a880a04ecf42e68da6d90c2b82508f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59cffaa5f3fe4ac79135e374c0668c4e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59d727b24fc44127a65031bd6fa0c440", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59e450f51e534a7e948a126aea9fc034", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59e77658f4db4341b138bbe31a9f6337", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/59ed19d81df244f0a1f03e91d8fcc576", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a00663d598042fcb8cdb4389bb3e1e0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a0b4f1b79ac4c5fa9918cbc7beb6b85", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a1c0a20cb2747158db69185705b1745", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a46bf7ec6db48e19b444308256eebae", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a4b2f8e22874e3cac100029686a1446", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a54b76854204feeb1615d31de2186c0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a5e2052a4e84fb9bb3c9160bf9ad512", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a6bfbb68031456880d637f9bb00ff32", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a6dbda72d18430eaf682ff9d1df3c20", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5a9b81bcd07b4bfa86f3db40d021e7ca", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5aa152f10353496c87ed6694c0c43fe1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5aad3d9eecc748dbb22acca2a775040c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ab3f13100be4b56b4fedb10333869d9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5aca4bf4402142eca648a05b6c1bf9ff", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5af462ce11fe4d36b74ad68e9e9cd45a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5b0a545ba2224f4187bad02ae7c111c7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5b1c3f4ec7e24c53bf5c574f982cffb3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5b3baa9fef744e65bd8c4bdee03f3d1d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5b7839de0e944ead96267515868fd26f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5b926eb583a941299cefee5c0c53b7d4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5bdf9e92a951409ab595a934b71ffc10", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c05aa6889344b40b04e35be5bcaa71c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c073d647c764584a3b6c5a819625f4c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c266e0ce56549329abd27d09eb8fd87", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c3bb58b54c94abc9dee1f29299df241", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c5f33eb869d414ebeabac1847babc30", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c5fc9246a284487bc2a1d92fec54ecc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c7dcfc5353e44828721d9b2b5157c05", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5c9cac845a2d47c494a644693e236ef9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ca7845f675a4a6daccc5c825e52452a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5cb2c4c516dc4635836b9af8fc061474", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ccbb76a5fe749e8b68a111feb3a9968", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5cdb3a0e343e4527bcef3372f9bb015e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5d247a058935496ea1cfa9786cb9c45c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5d6d935e28a9462faa0683ca00f52159", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5d87c549dc414a7b92d984d52575a810", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5dd5a00bd4e24363a1d11a0cb270496c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5dfb2de4c49941c79bfcf1f322962183", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5e129155168f4535a77257af474ef88f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5e3ec9e6ef914a12bf38eac58adf5f5a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5e4bc80b3eec4cee9eaf4c01fe78ca52", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5e693b6d600647729e443001c07eb84f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ea7abdfec0141048cf236587e967131", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ebdd321b32d4e88a2cfa1917bd37c85", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ec49631a8fe4d89a447a23248f273c8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ee4d284952d4fe0972a1a77ec8f2fee", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5ef00a98b9014203b345f3347709d4cd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5f04da0f009f4ca4b05e2ccd1bfb48a2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5f0f547bf6b04f5c8c2e488ca22e6b90", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5f31c75dcdf149c381a2f82540c48a4d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5f31d8be9d0545ba8d3b7f14ded24830", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5f5258515d8648cc817b7f88d4a26e62", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5f688d1ef03248169a76f6913af644b3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5fa39aa28ea8480db12ec5c6cf42b03f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5fa7190218944516af326fa663710adb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/5fb21161de074255b2af631b13dfe174", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/60060d3eb2ec41dea464a82d5d1f59fc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/603910fa2cc54f09ae283263c8d040e2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/603b5af9f85c4d65a9cc4379aa3de528", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6051f2a028b4416aa4a2dbf197688258", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/609dbcada7f7453e999187ad672caab4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/60b629080c574f83af3e7c0e48845219", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/60e5eddef93646318fd5efd76af74c67", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6105bf6ecf2f49bb89e72033ea1652c2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/613647f3913243509c0c515af2f1aa5b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/614601fe532a4a4aab83a1c48747973f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/61846c86fe0a4f86bf51d48141d7bc3d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6194adb68e9740c0ad9b1c526948b09f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/61d0a5fd8f3f4141b0fde96ceee9723a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/61eb251a0f3447428e36077330752104", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6201a979bacb4e6da9e37ed7063289dc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/621dd25cfd2145ada4d4bb26d3e2afbd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/62221ffbc4ca4c76b549b1bdb32fe524", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/624cc7c98c164a50a1e4860f08b418ca", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6259e655de494254831ae610433dd620", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/628dfda647fa4abd93f3c0f5f0870f56", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/62a514b463884c1c8aea022b52254e51", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/62ceca2b52fc452f817dcf90cb58c4cf", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/62d6a8c6e2a44fadb8d09a1c28c497dc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/630c32ec73ab4bdf90c3c16c3fa3a5b0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/632cd1414a5e4970ae5afda3f4530b39", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/63aea57d2f7a47b59c95433080a125db", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/63c9e072e37b4cd7be546cafb7dc4c4e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/63d84a237d5641dba0d06a27f4adb8a4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/640884f9cbbb4627b21192fc0bfc6dc4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6440a40128d84914ba1919dc7fe72c97", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/644fb96b8e3a41498e1b84b62bf88966", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/64840f7c87ec4fbca06ff4c57fe0d7e1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/649db95758504eb89c194e12e2e8fc2f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/64a17b7ddb9a4e0c847992fae537e559", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/64abd9e89e0b44e6b9dad0766dad888e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/64bd086f55a34fe1acb58dbc73c960be", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/64c9d9a9d3394196936db61cbcd08be5", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/64f98649cd594ebb8678ded56cbb9d23", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/650b4d9e7e6645888276a979680cd2b1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/65102a5a71c84bd69b8acedaa3208eeb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/651b203b2e0f436cb6d304825e374c7f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6538ff9201224bb99cd16dcfab5ff0a3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6563480e4ece40fa80e7e4ad79c98055", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/65b2490140114bb5b8e66cfd1752ca94", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/65c578bd33714b3385f8f0c68f63147b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/65eef4a06b3a48c089159155f6b1b723", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/65f72ba7563f4b3f8f007f6f60aadcab", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6622d412b0f84df78b8c3faad668340f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/66b588e81072453b85d893c5e07f2bdd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/66b75ccdff964d86a053e7667aa3c0de", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/66f95d9519994d1bbb4cbdb11ed5d3fe", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/671fe5222ec64f348dd84c59e4d608c8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/672be67356da4b8b9df8262dceeec85d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6757f81fc3f04e8e892efeedbaec05bc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/679c5056abf24513a82039270a04a991", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/67b3ef7dbf684acb879133ff45404b07", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/67b645b3ad0f443d89fd65f6b3eb2538", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/67ba62ac44a44c1983367ad2db9837b2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/67f0c0537a5a4befb61e82b0638c9edc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6804ccfc5bc14173a7035dad53c43dd4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/680d90027ae54bd9b8ea9375d27eb3ac", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/681c9105eb2444fca2012ee02e1d991c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/682785c10cce44e0adf3a30581b00094", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6833cdfe527a428e872ecb9bde13af90", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6860ce4f014d44b480a72e95040b4d48", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6860efc1a9d84aa3a41487a2da1c49d2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/687cfb0cdae3421dab71d8bdf9379fb6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/688387fd1b24461fb59b8abb1fae8d58", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/68860023b17b4979b827e1290358422a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/689d613c99bf472bb5d50d989e2d9552", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/68a2402232754adbab10ba8e6ecc8d51", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/68cb0f7ebb8a4e2aacfe13f7baf3aae9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/68e84f7db91142d684c11272d8f940bb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/691562b1cb3949de80f7c6e744a215a8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/696e3507bcd946849c7fd3a5343143ec", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/69ba166485cc4b77a92c6c8731175c59", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/69c4e86c59a74a548461ce04e69e1dd9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/69c8c8c8e1ad4872a9149fc9f926b2f6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/69ee0b7fafc2436ca7dd592ec99b5d99", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6a2e7b4fa4144078ae4549ca54c28b5d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6ab21545bd7e41b7b52c5cbde3c1b7e1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6adac89b900244e8848aee9f577982e7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6b02615899324307a1d023a05f84cbb6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6b583787f3e443cfa60f7c9e7c8e33c8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6b5b145cf88a413e9c2baa73cd0562f3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6b64a8d3407b41c785150e685b3ed784", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6b66c84565e642fe81b437e400ada642", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6b8f7c1a22814cadb245f25544a3691f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6bdf0d68507c47ebb6a7de6d5d4fe628", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6bdf175de55e4c81a6cbc3a59ef2f962", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6bf4623162594b468d29c26c44109c9d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6c22b0f03c6b4dc48e948c87fa6446f1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6c5a54959e6a478faa7170555147357b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6c7a54b28efc436cb1f17a21dc6b75b2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6cbb35fb9cdb426eabc3e81e7d782f07", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6d391b06fdf54605ad472e893d39f5db", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6d55a3c1a5094fb3b1beef4ee6eb74f3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6dbab4039c804257902e4c513ec04082", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6dc00f8cd6e642f3ad39458f4b1431c8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6dc1e3c486b1483f80cc70e550cd8684", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6ddfda2e329848e78b69cb8d70fbc72a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6dfc6f390cd54218835313b01aded5ef", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6e120862e2b1453aa7d167a17265fe44", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6e22d35e3cca4d999f77f4e0e103b436", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6e34785a22e1448b8408088d0eb61d75", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6e69aca77cae460a943b1d77895ef692", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6e6a3690a7424c26bca0b94a6ea32da0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6eb9bfb398be43cb8d8719fa37f88623", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6ef368ab8992435494df6d4b5cc0a459", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6ef6d0e4b8ba4a6d9b4fb8939f9273c8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f066b4e8e164adeb578cd52922eb5d8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f08c13b9dc044f5b533d998a686e973", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f259d93373f4e1c80deb50d1f69f310", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f36dff9a09a4de69cadc29adccffc28", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f379bf79523428982fc143fc6c1e92c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f3dcb234cfb4330a8b1d7c362e62dd0", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f578838ca534c2d92be7c1c6753031c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f5a8303c7124967bd3db29d5cfe187c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6f8cb3f622d340148e6f33c6bd2fcc68", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6fa57705f7bf4eb882cec9489ed49ff1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6fb10973211940638143e765ecd3c695", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6fc549b2a30e476ea1176dd41ad13e16", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/6fdb2b000bbf4cecb4736fa41cb0468d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/70488ba4454d4053a04eb837a7a79742", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7049b87f767345788c25ab9710737bcf", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/70a8c071d31b4cbb802ce33eed461b06", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/70b0a07069b14273b91057dd83603a00", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/70c02549bb3f4df0b352b252b085d1a7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/70e0203d53d94ee68ed89d452284525a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7121b89314304d95bb70ec4be746f1dd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/715f5b43745841319e1f34c89ef70330", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/716a563041ea42da93a4b679cfe44b8a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/71bf1b04ca72477d89814998869ed693", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/71f0449a0cb443c08084e3f1dd1d4373", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/71f9be1d8d89408195d38422bbfc2f4b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7238bb3270f34b4aaadd6979f3d8470c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7262f1e382c44f3185c9100d2753938d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/726ce320b3584cf8a28e817c3242dba1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/72be8fdd5ff74f0babf55ad149c5facf", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/733e50a9e14e4a628ecb1882f35c3a7d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/734da28741614718a6a3120a051fcd70", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7371dd7e71d44a049d59b7702981cf7d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/739256d50229438dbf22a9842aac502b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/73b72935ccdc4cfea5e4a59d797622bb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/73cf0b0ca2ff404292dab3496afbdc49", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/73e5d5dd790547beb83f68374bb540ae", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/741cbdd9a8524bb0b923a05876b5dc5e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/745022626b57487bb9e99e93594852b4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7458a0181b954cbca3558ba8fc357999", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/746eb496bd664262a6c711c01b6d8bda", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/74811dd4c9e74e979b7e368abd4dc8d4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7482f0782fd546dd91896f250f100120", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/749b09c064a04bf78d7e5b49a6878503", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/74b4648d820c4b80abad6400e40ac0d2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/74c4e22e901a4d819285b0ac306306ad", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/74df67058e69472eb24127e3af06137d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/75234d7de75d4294bdf8fd3f9c079ca6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/75380057accb4df4b38e006a525e058c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/755b191d3b814761a3802960698bd497", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/75920200d2ea4ea39f6ac2323b59295c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/75d58fe52c664f3eaf64ce217bd9fc37", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/75e8d556f8fe4a6abec09d5e5752ba65", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/75fd658580fa40d8925fdf2eceba074c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/76ae5b49581a4e26a4f341a3f87d9194", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/76b57d1dc7974a42943aa3b80a43f1b3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/76c965338d594813b00e1fd8209d79c1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/76e3abd380854e2bbc83956d9475b221", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/76ecf100cec84c4ca1084d2ce67715dd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/77c6bc7233c640b983ef2ad0beca252f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/77cdabcc5c2c4391a7c511939d3b4b85", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/77d0bbaa6ae34d8181e05cfca4cf4467", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/77d423301c00484c9cc27dc821ff21ac", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/782589ccd8db4b61ab4606abff30af69", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/786a5c0d150b4f9fbbd12fc5e119a66a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/789decb0bc794e2ea1f1949cfd6353fd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/78dcbf6c06454cb1a6057c82b1e8b19b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/78f2aeb3d3644796a2f3f8be1abae0ae", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/78f99003441c47f78b0507964cd87b9f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/790e2909f49d4d85bfbcf3e83c764c39", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7946d9855cf74974900481b8810c02cd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/79621610a99e4b769cd9868ece807f20", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7988e57a850b40e7acd23e444cca15fa", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/798d58590557451e89a931450afa08af", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/798f042094284fd3b944ad96776323a9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/79a26b8778f44d22ae6ac0a98b611ea2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/79af6de90e79471ab0d0dba4b9f63231", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/79b14b4ea29442cfa79bda69b9b718ce", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/79e8b3d0bd2e4513ad69b7e21a7d2f1b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7a444a9d23aa488bbcba4416b5075c64", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7a590033cd0b46db925169457b10e249", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7a59d1ead80f460c9331c4883ba8db00", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7a6e92b68a4c417ab55fa0b47d0a1957", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7aa8b2e4c2744047b71b7c3b006954b3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7ab10d6d741949cdaa13dce60534481d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7ab52af3cdcc4dfbb389a2574443ec1a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7abdde8e6ed6432b96389a6d6fa9f46e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b003cbfc4dd4fb9a64e30d0d3aa0fb6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b0d8b216a91467a91d600a4febbb5bb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b126f84490241e5b32061f1c65ec4f9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b20178924244999b948725a446c31f9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b32694127d34d178297a8e1d415ab7c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b86acff236b41d1900fbf03b4423c1b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7b88f69b5c2f46d7a8c5cc9d0df76bf8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7baa874045aa40d2bcfd4893d24af07c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7bb1f9540eb045e1a2eb891384582426", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7bc786b231344685b93f9fb4195b74bc", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7be8c3d8343644b1a9876055fc62654d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7bee652ee0494b5cbb2a988237df0203", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7c13912d2447406898a921feaae9e2b7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7c3804791ea84a9da99986ad49f2cf16", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7c584f68bdc7488f80df60e429fce654", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7c642e24d8c147f5ab7f70ba68c6af55", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7c7e8f27018941b492f99889b7ed1896", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7cb2e4b5097f44cab09348ecaf20971b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7cb539e06b7b4ddfa49b6a87d8f92989", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7cbdab0fa54943fbb419ad5683f4c86c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7d0c05d1e5114e12b55cbd8b700f7398", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7d45941853654441be59e2f0a91888f9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7d986269d30949229ba6c1914b2a51ee", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e12755198da4917bb556e65908ce971", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e1726ce54c04599b2122a70a7fb7b1a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e22bd36d11e4a7889380f3ec4fdcf51", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e53fb17eb164b7cbf4c1fc72ff385ca", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e5fa6808c7a4cff997186b4b83f1604", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e63bf654ab347c980c81d788401c535", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e64f14372cf4caeb3dc956ea5106b9c", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7e691053a19549d2822089a2de413af2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7ea1f503c0e740058149e3698cc2edbb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7ea4a6049ca7465c8e70780fce724d1d", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7eb05c8b1c524f40b262bc2cb6ef4e56", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7ee891771ef04224abe47e98ffa21d96", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7eea6fe6f3de478db61733a7e9257b74", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7f231bddb9cd44a28d52c4dbdf8ba87f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7f4a217986d245b09d87a518d25b69ff", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7f8433d843f94d059b56825f77aae456", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7f903577fd9642ab945b0cf5aaedb0ed", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7fa8807e8b0949c4ac8099ba3bb48211", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/7fd72c2c91b44093a39d6ba7d1fa08d2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8013ccae00424a6bb4bf98203f0dadd9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8032d60b76d1428fa8baac54d7d961b8", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/80444d0c61dd40ee9de8e52db4bf65e2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/80715fe35da94e2c989c49a9a37b6559", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8126f07506864d1a85f06a575329d4b2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81312f5eebe94ee981f397ae73294549", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/819c8885b7ec4f38bbe748a1d03085e1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/819ee2e239414fb1aab29c9eeb8f1cd3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81a70846106c4518a2809ff3772d04bb", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81aab5dfc5f544309836db85b91d22de", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81c494a1a5384b47a22220dd3418ff63", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81dc6e5ca4ac42dab2371c2113275451", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81f89f51bf204aabba75dc1826985fee", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/81fc3b92aa5947acb907d8871d99577f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8209864084bc4f78b22bd17bc3e772de", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8219df04815a499f97e5f0f3d797ef3f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8220092870994e139d65aff7c09a7926", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/825fc37b3890428c98c8e5b3ba5a06ca", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/827822640854465ba70dcb3f99fc9bf4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8287cee809794211ab080168ca128888", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/82aa82a7081e4d1a9f318820dc20357f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/82c229f5c21446e0a9a2a8124351bd63", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/82d8336a6e0d4aa0b505db9311906b65", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/82ee0ab8ef86400ca4af2d117b72aa02", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/82ee2edb101146fda84fea12cc5f87bd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8308a459aef6434c8f026cab1195b656", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/83177fcc5cfb489aa0282dfcef05486b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8318c21efb9146b4b76241e526a300f6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/835036b46edb4fc798d7077202b3874e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8351e4614a664c09ae1c8bf6401ad196", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/83599e4f6a4142c0a125dd8a1f52b18f", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/836d08125a3a422686957cdf35d2a455", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8370656863e04ac68a464e0f34fbe39e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/838b78a508134f46bf403abda942d87a", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/839168757a784299a53a2cd9b4ed3aab", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/841c6e652f6240daa5166429e09afa13", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/842aed6700dd41698a73c3fd3181c117", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/84310d7030354ddc82a6468a3c02244e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8467e1051da54e95aa7dcdaa41100716", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/847b75b555564a8e8210c3be0e5eb734", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/84cd5ed2da0442fb8b4600deb9158323", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/84f922542af447c59d32facafd823916", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/85297b7ba1a145018b2a2d5f0c695f67", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/853845603f6346b9acd7f64fd8f44c49", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8550ded8935f4aefbc745a5bfb05e513", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/856c1c329411461e85dd24e8fdf6ad80", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/85bd50ae9bf045288094ae165cd53dfa", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/85bdfe1e275a4b59940c49dcf3a0103b", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/85d76049fe05424eb8fc52cc9370c4f2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/85f5cbd479064b318aa0fd5c46edceb4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/86095fcff297481d8b4361d5458517c6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8662140044fa4f029bc1feeca19b0178", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8666c09ad10b41108ae85a19ee389626", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/868c68371caa433397c41f54008d39fd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/86cbaf53d22b4505a5d5f1079f0b0bfd", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/86e650efe56f443793b5e6f300cc0985", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/870241720cac4d8ea64acdd681e3f525", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/8721568fc9b9467db6928311952e2ab7", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/87395a3e76aa4968a01e1f63c183bfc9", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/877c423669c5491cb393b8c63267a7a6", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/879d24db5bb2433b8ded073a6cef1173", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/87cb51928edd4a34b533ef201f61704e", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/87e58a7867a2410eac560730e4ceb4e1", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/881cc59b19b14fddbee8e898bfa045d2", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/881eb764f2fc499e934b0beed962bbd3", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/884788e98c5643e6b1a10562c3d62783", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/888a4f545217477ca99e6cb44a60f8f4", + "attachment_id": null, + "used": false, + "result": "planned" + }, + { + "key": "o/88b2f6e0d0084c10b1ed1a1b86596342", + "attachment_id": null, + "used": false, + "result": "planned" + } + ], + "restored_object": [ + { + "key": "o/77f4fbc630a1966cb3f6d4e09c019318", + "attachment_id": "ed0ff4c7-507b-44cd-8928-622ea6e977a8", + "used": false, + "result": "planned" + }, + { + "key": "o/ee7f3d21bebb4a56572b9b1f7d2299ee", + "attachment_id": "4c72dcfe-c46f-47a0-a442-2937c722ed99", + "used": false, + "result": "planned" + } + ] + } +} \ No newline at end of file diff --git a/evidence/reconcile/interrupted-2-rerun.json b/evidence/reconcile/interrupted-2-rerun.json new file mode 100644 index 00000000..da828519 --- /dev/null +++ b/evidence/reconcile/interrupted-2-rerun.json @@ -0,0 +1,6055 @@ +{ + "mode": "apply", + "grace_seconds": 30.0, + "verify_checksums": false, + "duration_seconds": 12.891, + "error": null, + "scanned": { + "objects": 1746, + "rows": 186, + "consistent": 184 + }, + "findings": { + "orphan_object": 1560, + "stale_intent": 0, + "missing_object": 0, + "size_mismatch": 0, + "checksum_mismatch": 0, + "restored_object": 2, + "fresh_orphan_object": 0, + "fresh_intent": 0, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 0, + "actions": { + "planned": 1562, + "applied": 1562, + "skipped": 0, + "failed": 0 + }, + "cost": { + "s3_list_requests": 2, + "s3_head_requests": 1562, + "s3_get_bytes": 0, + "db_pages": 1, + "page_size": 1000 + }, + "samples": { + "orphan_object": [ + { + "key": "o/3fcfc7880b8f4924b8e1a16164af8980", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/406dcde1f0ff46b7b15139434125cb89", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/40b43f8a2bf04b68b377c1e0a6ecf525", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/40e497e353cd4215af041946a0c26b9b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4105740884cd4215b0f66c4d8f4ab681", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/411289b7102a4e76a6dcce99c280cb27", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/41357c3da64845f481af9c1ef299ea9e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/414e6e3ed1da435085230fede8a3be95", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/41a5bff616aa42c689e2ee0753d036bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/41f49bb8f0314f5ab8e3b33f1cb98066", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/420b27329f6f4162ba1fa98bcb1396b5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/42238335bb9f43a6ab94d1a0587f4919", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4224f05c869446c7a2425394471843f0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/423d034b5693478c9c72d0f9cc3a313f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/42597cde33fc40799c5a94a3da77b2d3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4266e0c5c4aa4635971802d33e4e6242", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/428a288d12a74f6ba33371be745c1533", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/42a3e929a72f40f3ad4e0121ca5f01d5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/42a961603d544c3ca214701870219655", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/43120578859d42f6974e0274c7188b95", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/431d3d3db964453aad41ff0ec3babed7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4344d5e732d74ad8a1bb5ffcfd135b62", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/437fba8edba64c6dbb4e93a34f90fd0e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/43862bc845644860aa847b2a584fcb7e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/438f9d52a0a84d4db2533dad88e0605f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4392b7187bca48dc913fde23989c9e4b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/43b8db793f3545da9fb5fa38e2d4ec43", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/43eca42668684e54aafe37e77051e2ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/441535cceb244018997ed9d70c7054df", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/44310108170443bebcef3a8aea90c606", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/443aa2c6f6fd4c06a34ddf66a0ea3200", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/444a98dfa5d5494d8007dc892e80f296", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/44581d4ffc104eb6a0bcfe1cb364fbef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4466f206dcd84d0f84a699a5cc5ca90d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/44826520ad1b4adba667336a92f22889", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4494042a5ce84e69bf48de1f83689300", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/44988e38804042dbbef3a1f38042cd93", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/451160b197b242d091bb42cdfd1722e2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4512223d967643608842959696f7a5d8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/453201b972534a17a3731d27b33a6eaa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/453d3b0b07ad4102b8fe9244710260b0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4563874dad62475f8cf36e707bcfda29", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/456f527e2d5b4eaeb4a7fbe813dc9f0a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/45c32b716181481f9df377a22f3579b4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/45d4bd587d6e4865932cb9f1780fff0c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/45d4d80e9e67473ebe633fde6f74ea2a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/469af1343b2448c392126f420f0734c7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/46af780ad90348c6a44a1849ba3b5045", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/46b4a8edd3fe4c33a82e5acfcd3161fa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/46dd9e4430144d8aa183a3e7e377ac7e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/46e8b7b5bd184e1fba285fcfe49b21a7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/46eea46534f94489a1ca97a4cc58ab50", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/46fad16980e54d30842cf0b7c53be4ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4705f8b4399842a5b2e3a3237b0d8f94", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/47064db5672047d191aa348e20169b31", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/47356bb6ddfe438a8837119329c7073b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4737ed9e731f4c4ba200c064e1114013", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/476e49019f76450f81fe6826381fede2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4776386343b347628f1499281b5a0fbf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/477a11a3c5e34e54a85099b91f4a9bb6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/477c392e4aa54f999399fe867bdadd94", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/477d5d557d484cdf9ec9f2f7d348e17b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/47809e39f893440391439cf50c4984ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4784de5611c84e1cb5c031df06cc206c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/47ac6af4d1304fbda0ba9423f861cb88", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/47bb74c16a4c42ec9d45eaf71f0a1d67", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/47d9b07a8a5747e99d53f8cac029bd89", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4832d4af593e44faa8a8a38acc2e555a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/48a5cd7b296c406f895db5eb5c44cfc9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/48ba443e37be4bddad5c174d5020982e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/48ea081b62314d9496e1f9043ec37bbc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/495ddab1778b4217aac9a5dcd6857285", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/497c3ac93bfc4b7e8e369c89240019f0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4992a3e1eae547dd8646afa1424ded14", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/49af3ac368be42a4900eebb3b1d2c6c7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/49dfdf4d96654021914301166e65ff73", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/49f3693bdcac43f1b50aabdff556a6de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4a1f01aa66c1493f80cca5dc7f09e001", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4a6435baf3ac4d78a3234e16897319da", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4ab4f021a89b4ac698935be7d035e43f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4ad69eb583784a37a6874dbae0a1a8d0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4adb920affbc437fa0e09b3f28557788", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4aff7b79e80d4b38894e1f6f609b7185", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4b195cdfd9ac47119b228065022e5c14", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4b214e27c9f2497db89a0a186b1913ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4b3948a665694e3186ecab562221213d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4b63717ab4754837ba17b887085ccd46", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4bd71ab669a9416c8efeb78c782d3648", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4bf9a3ccee2c42e4b4ca81cda3e3b54c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4c107266825045d2976d9a7a3a5dc99a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4c14b5683b8b4b20aab0571410f4283a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4c442b49d7634d918eb3864a2f6bb077", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4c7ef97aca9940bb997289303e911827", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4ca451c259664da8aa6324a1fb5ac495", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4cba8495b7a545d2a9d2c556b05a4f1e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4cc66ff9380d456cb332b4181f165818", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4cea96a4f7e1439a8850d539f10dc2c4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4cffe16bc06c46a5904e9117b2298da2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4d16b2ca229f4f9cbf2ce03c08be02f3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4d19768d78eb4b9b998cacc95843f274", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4d5c8e4d17eb4ebe93b7c7199c9b6029", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4d5df3d192a84c02b93ebbee86732255", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4d7d0059c3ae430db1b43c65fe511242", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4d7f812d745e4ab38eb2f41095df3b0b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4db91e89937d42eeabcbb678ac67ff8a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4dba51d2bc3e48d999f39369f7efd02f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4de8ff0d033748a0b05cd1bbcf9a6616", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4df8857f0644447981c844c57a1210c6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4dfb636e2de74f2d873c776eb218ebf3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e060f3b6b1d45f9a4905c86963f66fa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e0bb1ed5ec14637bb106b3a159bdc37", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e0e52c36d2442099a1662c8e0e876d4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e2a837216154f40b025bc1869fffa11", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e79a33ccf2a4c04be4386ce902ff4bf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e7f820d19ca4b2180504679d610a5bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4e8b6f2f0b9a4d7d8bb1f39e4a2a045a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4f0a2d3c515f41d28ad519c594b590ba", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4fa94bdfbe7e4ff1abb6b7b6adc4bdef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/4fc7f33a0f8b43f0b9aa35c87545628a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/50425f113d1d4551a9bc5096df572ddd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5055979b15014dc098a9ad8c7f187bd6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/505b298298b540419d7bf408ba92e317", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/50b3966f373a4767942fb2762250467c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/50e80ec2ef4b435883331049daba3dda", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/50fa7bd8fd164bfd918d6cc2b4128ff3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5106c60032be46679aa4c84582c447e8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5119ed50d87142648f40af6d270dee6d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/514e5d12d6594d258bf85a6f3d01ee99", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/51ebe5b069b14d71b09b1dc086ae5edb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/520a5a489f5049089cc9b64d95726c03", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/520eb1db7bd948c3a5b0907034563a39", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5220b75d311541e0adf6e6b39b0b8aa7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/525f4237bf634b8b8c3b3a531c206194", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52678e8f2b474c98b788be0a892be815", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/529d8d7b43e641d29cada8d2fa08d17a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52a437c3035c4b23bb1a946048e2c7ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52abf0c22a1a4bbfb7cd6e1cbeed6a38", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52cca875da374e0a9efafe3061c9cabb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52df06c3f08b482cae786ca09c937ff7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52e5cd5af3884254907857babcd48677", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/52ead265e5e54fcb9edaddb28bf70618", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/53037f957b2e41f0ac1c5408fa554b6f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/53538d063a114b0395943612ecc8e710", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5387b490ade1496fbe814b13e271072b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/538933eb554843d0af04363ab8adfb81", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5398c9e1cd3d420e933dfd9911127279", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/539d4a8797264a9d8d999f5b2ce4ee9f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/54071bb6fe6048a086107c56f6d3d50a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/541648589d224a7bb76976f91a4c0052", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/546ce56bc32f4854bad49c93bfa1a803", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/546f8c8c7a7b4962a95ef2d268b17d38", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/54a815a4ca444a0f924e3d0bac58efad", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/54b2144f0cf2485cbaf824583c98fe24", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/54f616cf3615413da2ee9936b87f9da5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/54f688e7ca77476a9394489878f31188", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/550e006e700e4d8ca73fb4f46bf0ad36", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5512a26bebfd433ea95c4eea01524655", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5512a74cdd484e40b4c9abd6e316797d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5524b4d455b24c93b6ad3a6899f420b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/553f539a2d7d4068b02485ba0d89a603", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/55626174b0fe4263abaeae54e3450b45", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/55687f2648f645d587153f009d1c4158", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/55708ba805954202a36d5ee0282a0cba", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/557bfe6b61d34392b29569491c51faf7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/558fe4d4d6fb448097a8849cd3f638d4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/55e8d2baad8645898e751d5eae762cf3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/55f7465e1666447389b7c513cd1739bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/56139b94586d4839b904bf70b5744d2d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/564756548d2b4d6082129155919b241b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/56cb1b27997343a1be03e3fdc4284bf6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/56ec0a8ea721453ebca2cbe10ffbaf0f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/56f3c3d406374c70a53931ae34751c52", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/56f457afb8974ffaab760bdee663a90a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/57182ae676ba4d5090184397f063d2fd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/57394346e0cb48358513718b2e0a3288", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5749abe433824c2ca8116e2c796e96df", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/57a4377354844c91bb110b5f336422de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/57ac031447a04e9f87c8c7c451a135f8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/57b566bec95541e1ad1509299592e942", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/57ba5a92424d445d9b7370b817509289", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/580b68885eed498ea79cb910b9326aa4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/58177778ae7446bb9a862ed8d08afbec", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5836b13b18504596a670be13f239c9fd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/585ee17ea6c44f5d842fb29abd3153d7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/58f5b284bf61461ca15ee288b64130d3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/590bbe0c2d3b473faa83503e0aa6cd06", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5936b6a7f9c04056a14e38dcf3346b98", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/594b7690533f40d285b33d889869cdd4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5970a038263c4d3f91239085efbf1558", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5972e161c2c1464b95d5016580f50ea4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/599c7604efc542d98311258fc02e87cc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59a5b1dfd7e1426280ee362bfe1859cf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59a880a04ecf42e68da6d90c2b82508f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59cffaa5f3fe4ac79135e374c0668c4e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59d727b24fc44127a65031bd6fa0c440", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59e450f51e534a7e948a126aea9fc034", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59e77658f4db4341b138bbe31a9f6337", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/59ed19d81df244f0a1f03e91d8fcc576", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a00663d598042fcb8cdb4389bb3e1e0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a0b4f1b79ac4c5fa9918cbc7beb6b85", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a1c0a20cb2747158db69185705b1745", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a46bf7ec6db48e19b444308256eebae", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a4b2f8e22874e3cac100029686a1446", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a54b76854204feeb1615d31de2186c0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a5e2052a4e84fb9bb3c9160bf9ad512", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a6bfbb68031456880d637f9bb00ff32", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a6dbda72d18430eaf682ff9d1df3c20", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5a9b81bcd07b4bfa86f3db40d021e7ca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5aa152f10353496c87ed6694c0c43fe1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5aad3d9eecc748dbb22acca2a775040c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ab3f13100be4b56b4fedb10333869d9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5aca4bf4402142eca648a05b6c1bf9ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5af462ce11fe4d36b74ad68e9e9cd45a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5b0a545ba2224f4187bad02ae7c111c7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5b1c3f4ec7e24c53bf5c574f982cffb3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5b3baa9fef744e65bd8c4bdee03f3d1d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5b7839de0e944ead96267515868fd26f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5b926eb583a941299cefee5c0c53b7d4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5bdf9e92a951409ab595a934b71ffc10", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c05aa6889344b40b04e35be5bcaa71c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c073d647c764584a3b6c5a819625f4c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c266e0ce56549329abd27d09eb8fd87", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c3bb58b54c94abc9dee1f29299df241", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c5f33eb869d414ebeabac1847babc30", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c5fc9246a284487bc2a1d92fec54ecc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c7dcfc5353e44828721d9b2b5157c05", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5c9cac845a2d47c494a644693e236ef9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ca7845f675a4a6daccc5c825e52452a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5cb2c4c516dc4635836b9af8fc061474", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ccbb76a5fe749e8b68a111feb3a9968", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5cdb3a0e343e4527bcef3372f9bb015e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5d247a058935496ea1cfa9786cb9c45c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5d6d935e28a9462faa0683ca00f52159", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5d87c549dc414a7b92d984d52575a810", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5dd5a00bd4e24363a1d11a0cb270496c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5dfb2de4c49941c79bfcf1f322962183", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5e129155168f4535a77257af474ef88f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5e3ec9e6ef914a12bf38eac58adf5f5a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5e4bc80b3eec4cee9eaf4c01fe78ca52", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5e693b6d600647729e443001c07eb84f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ea7abdfec0141048cf236587e967131", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ebdd321b32d4e88a2cfa1917bd37c85", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ec49631a8fe4d89a447a23248f273c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ee4d284952d4fe0972a1a77ec8f2fee", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5ef00a98b9014203b345f3347709d4cd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5f04da0f009f4ca4b05e2ccd1bfb48a2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5f0f547bf6b04f5c8c2e488ca22e6b90", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5f31c75dcdf149c381a2f82540c48a4d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5f31d8be9d0545ba8d3b7f14ded24830", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5f5258515d8648cc817b7f88d4a26e62", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5f688d1ef03248169a76f6913af644b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5fa39aa28ea8480db12ec5c6cf42b03f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5fa7190218944516af326fa663710adb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/5fb21161de074255b2af631b13dfe174", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/60060d3eb2ec41dea464a82d5d1f59fc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/603910fa2cc54f09ae283263c8d040e2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/603b5af9f85c4d65a9cc4379aa3de528", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6051f2a028b4416aa4a2dbf197688258", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/609dbcada7f7453e999187ad672caab4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/60b629080c574f83af3e7c0e48845219", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/60e5eddef93646318fd5efd76af74c67", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6105bf6ecf2f49bb89e72033ea1652c2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/613647f3913243509c0c515af2f1aa5b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/614601fe532a4a4aab83a1c48747973f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/61846c86fe0a4f86bf51d48141d7bc3d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6194adb68e9740c0ad9b1c526948b09f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/61d0a5fd8f3f4141b0fde96ceee9723a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/61eb251a0f3447428e36077330752104", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6201a979bacb4e6da9e37ed7063289dc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/621dd25cfd2145ada4d4bb26d3e2afbd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/62221ffbc4ca4c76b549b1bdb32fe524", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/624cc7c98c164a50a1e4860f08b418ca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6259e655de494254831ae610433dd620", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/628dfda647fa4abd93f3c0f5f0870f56", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/62a514b463884c1c8aea022b52254e51", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/62ceca2b52fc452f817dcf90cb58c4cf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/62d6a8c6e2a44fadb8d09a1c28c497dc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/630c32ec73ab4bdf90c3c16c3fa3a5b0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/632cd1414a5e4970ae5afda3f4530b39", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/63aea57d2f7a47b59c95433080a125db", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/63c9e072e37b4cd7be546cafb7dc4c4e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/63d84a237d5641dba0d06a27f4adb8a4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/640884f9cbbb4627b21192fc0bfc6dc4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6440a40128d84914ba1919dc7fe72c97", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/644fb96b8e3a41498e1b84b62bf88966", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/64840f7c87ec4fbca06ff4c57fe0d7e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/649db95758504eb89c194e12e2e8fc2f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/64a17b7ddb9a4e0c847992fae537e559", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/64abd9e89e0b44e6b9dad0766dad888e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/64bd086f55a34fe1acb58dbc73c960be", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/64c9d9a9d3394196936db61cbcd08be5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/64f98649cd594ebb8678ded56cbb9d23", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/650b4d9e7e6645888276a979680cd2b1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/65102a5a71c84bd69b8acedaa3208eeb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/651b203b2e0f436cb6d304825e374c7f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6538ff9201224bb99cd16dcfab5ff0a3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6563480e4ece40fa80e7e4ad79c98055", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/65b2490140114bb5b8e66cfd1752ca94", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/65c578bd33714b3385f8f0c68f63147b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/65eef4a06b3a48c089159155f6b1b723", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/65f72ba7563f4b3f8f007f6f60aadcab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6622d412b0f84df78b8c3faad668340f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/66b588e81072453b85d893c5e07f2bdd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/66b75ccdff964d86a053e7667aa3c0de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/66f95d9519994d1bbb4cbdb11ed5d3fe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/671fe5222ec64f348dd84c59e4d608c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/672be67356da4b8b9df8262dceeec85d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6757f81fc3f04e8e892efeedbaec05bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/679c5056abf24513a82039270a04a991", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/67b3ef7dbf684acb879133ff45404b07", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/67b645b3ad0f443d89fd65f6b3eb2538", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/67ba62ac44a44c1983367ad2db9837b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/67f0c0537a5a4befb61e82b0638c9edc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6804ccfc5bc14173a7035dad53c43dd4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/680d90027ae54bd9b8ea9375d27eb3ac", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/681c9105eb2444fca2012ee02e1d991c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/682785c10cce44e0adf3a30581b00094", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6833cdfe527a428e872ecb9bde13af90", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6860ce4f014d44b480a72e95040b4d48", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6860efc1a9d84aa3a41487a2da1c49d2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/687cfb0cdae3421dab71d8bdf9379fb6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/688387fd1b24461fb59b8abb1fae8d58", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/68860023b17b4979b827e1290358422a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/689d613c99bf472bb5d50d989e2d9552", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/68a2402232754adbab10ba8e6ecc8d51", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/68cb0f7ebb8a4e2aacfe13f7baf3aae9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/68e84f7db91142d684c11272d8f940bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/691562b1cb3949de80f7c6e744a215a8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/696e3507bcd946849c7fd3a5343143ec", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/69ba166485cc4b77a92c6c8731175c59", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/69c4e86c59a74a548461ce04e69e1dd9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/69c8c8c8e1ad4872a9149fc9f926b2f6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/69ee0b7fafc2436ca7dd592ec99b5d99", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6a2e7b4fa4144078ae4549ca54c28b5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6ab21545bd7e41b7b52c5cbde3c1b7e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6adac89b900244e8848aee9f577982e7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6b02615899324307a1d023a05f84cbb6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6b583787f3e443cfa60f7c9e7c8e33c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6b5b145cf88a413e9c2baa73cd0562f3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6b64a8d3407b41c785150e685b3ed784", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6b66c84565e642fe81b437e400ada642", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6b8f7c1a22814cadb245f25544a3691f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6bdf0d68507c47ebb6a7de6d5d4fe628", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6bdf175de55e4c81a6cbc3a59ef2f962", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6bf4623162594b468d29c26c44109c9d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6c22b0f03c6b4dc48e948c87fa6446f1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6c5a54959e6a478faa7170555147357b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6c7a54b28efc436cb1f17a21dc6b75b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6cbb35fb9cdb426eabc3e81e7d782f07", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6d391b06fdf54605ad472e893d39f5db", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6d55a3c1a5094fb3b1beef4ee6eb74f3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6dbab4039c804257902e4c513ec04082", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6dc00f8cd6e642f3ad39458f4b1431c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6dc1e3c486b1483f80cc70e550cd8684", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6ddfda2e329848e78b69cb8d70fbc72a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6dfc6f390cd54218835313b01aded5ef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6e120862e2b1453aa7d167a17265fe44", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6e22d35e3cca4d999f77f4e0e103b436", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6e34785a22e1448b8408088d0eb61d75", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6e69aca77cae460a943b1d77895ef692", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6e6a3690a7424c26bca0b94a6ea32da0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6eb9bfb398be43cb8d8719fa37f88623", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6ef368ab8992435494df6d4b5cc0a459", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6ef6d0e4b8ba4a6d9b4fb8939f9273c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f066b4e8e164adeb578cd52922eb5d8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f08c13b9dc044f5b533d998a686e973", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f259d93373f4e1c80deb50d1f69f310", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f36dff9a09a4de69cadc29adccffc28", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f379bf79523428982fc143fc6c1e92c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f3dcb234cfb4330a8b1d7c362e62dd0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f578838ca534c2d92be7c1c6753031c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f5a8303c7124967bd3db29d5cfe187c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6f8cb3f622d340148e6f33c6bd2fcc68", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6fa57705f7bf4eb882cec9489ed49ff1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6fb10973211940638143e765ecd3c695", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6fc549b2a30e476ea1176dd41ad13e16", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/6fdb2b000bbf4cecb4736fa41cb0468d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/70488ba4454d4053a04eb837a7a79742", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7049b87f767345788c25ab9710737bcf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/70a8c071d31b4cbb802ce33eed461b06", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/70b0a07069b14273b91057dd83603a00", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/70c02549bb3f4df0b352b252b085d1a7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/70e0203d53d94ee68ed89d452284525a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7121b89314304d95bb70ec4be746f1dd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/715f5b43745841319e1f34c89ef70330", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/716a563041ea42da93a4b679cfe44b8a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/71bf1b04ca72477d89814998869ed693", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/71f0449a0cb443c08084e3f1dd1d4373", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/71f9be1d8d89408195d38422bbfc2f4b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7238bb3270f34b4aaadd6979f3d8470c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7262f1e382c44f3185c9100d2753938d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/726ce320b3584cf8a28e817c3242dba1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/72be8fdd5ff74f0babf55ad149c5facf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/733e50a9e14e4a628ecb1882f35c3a7d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/734da28741614718a6a3120a051fcd70", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7371dd7e71d44a049d59b7702981cf7d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/739256d50229438dbf22a9842aac502b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/73b72935ccdc4cfea5e4a59d797622bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/73cf0b0ca2ff404292dab3496afbdc49", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/73e5d5dd790547beb83f68374bb540ae", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/741cbdd9a8524bb0b923a05876b5dc5e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/745022626b57487bb9e99e93594852b4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7458a0181b954cbca3558ba8fc357999", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/746eb496bd664262a6c711c01b6d8bda", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/74811dd4c9e74e979b7e368abd4dc8d4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7482f0782fd546dd91896f250f100120", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/749b09c064a04bf78d7e5b49a6878503", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/74b4648d820c4b80abad6400e40ac0d2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/74c4e22e901a4d819285b0ac306306ad", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/74df67058e69472eb24127e3af06137d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/75234d7de75d4294bdf8fd3f9c079ca6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/75380057accb4df4b38e006a525e058c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/755b191d3b814761a3802960698bd497", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/75920200d2ea4ea39f6ac2323b59295c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/75d58fe52c664f3eaf64ce217bd9fc37", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/75e8d556f8fe4a6abec09d5e5752ba65", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/75fd658580fa40d8925fdf2eceba074c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/76ae5b49581a4e26a4f341a3f87d9194", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/76b57d1dc7974a42943aa3b80a43f1b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/76c965338d594813b00e1fd8209d79c1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/76e3abd380854e2bbc83956d9475b221", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/76ecf100cec84c4ca1084d2ce67715dd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/77c6bc7233c640b983ef2ad0beca252f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/77cdabcc5c2c4391a7c511939d3b4b85", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/77d0bbaa6ae34d8181e05cfca4cf4467", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/77d423301c00484c9cc27dc821ff21ac", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/782589ccd8db4b61ab4606abff30af69", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/786a5c0d150b4f9fbbd12fc5e119a66a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/789decb0bc794e2ea1f1949cfd6353fd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/78dcbf6c06454cb1a6057c82b1e8b19b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/78f2aeb3d3644796a2f3f8be1abae0ae", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/78f99003441c47f78b0507964cd87b9f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/790e2909f49d4d85bfbcf3e83c764c39", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7946d9855cf74974900481b8810c02cd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/79621610a99e4b769cd9868ece807f20", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7988e57a850b40e7acd23e444cca15fa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/798d58590557451e89a931450afa08af", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/798f042094284fd3b944ad96776323a9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/79a26b8778f44d22ae6ac0a98b611ea2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/79af6de90e79471ab0d0dba4b9f63231", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/79b14b4ea29442cfa79bda69b9b718ce", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/79e8b3d0bd2e4513ad69b7e21a7d2f1b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7a444a9d23aa488bbcba4416b5075c64", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7a590033cd0b46db925169457b10e249", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7a59d1ead80f460c9331c4883ba8db00", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7a6e92b68a4c417ab55fa0b47d0a1957", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7aa8b2e4c2744047b71b7c3b006954b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7ab10d6d741949cdaa13dce60534481d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7ab52af3cdcc4dfbb389a2574443ec1a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7abdde8e6ed6432b96389a6d6fa9f46e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b003cbfc4dd4fb9a64e30d0d3aa0fb6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b0d8b216a91467a91d600a4febbb5bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b126f84490241e5b32061f1c65ec4f9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b20178924244999b948725a446c31f9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b32694127d34d178297a8e1d415ab7c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b86acff236b41d1900fbf03b4423c1b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7b88f69b5c2f46d7a8c5cc9d0df76bf8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7baa874045aa40d2bcfd4893d24af07c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7bb1f9540eb045e1a2eb891384582426", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7bc786b231344685b93f9fb4195b74bc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7be8c3d8343644b1a9876055fc62654d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7bee652ee0494b5cbb2a988237df0203", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7c13912d2447406898a921feaae9e2b7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7c3804791ea84a9da99986ad49f2cf16", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7c584f68bdc7488f80df60e429fce654", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7c642e24d8c147f5ab7f70ba68c6af55", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7c7e8f27018941b492f99889b7ed1896", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7cb2e4b5097f44cab09348ecaf20971b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7cb539e06b7b4ddfa49b6a87d8f92989", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7cbdab0fa54943fbb419ad5683f4c86c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7d0c05d1e5114e12b55cbd8b700f7398", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7d45941853654441be59e2f0a91888f9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7d986269d30949229ba6c1914b2a51ee", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e12755198da4917bb556e65908ce971", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e1726ce54c04599b2122a70a7fb7b1a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e22bd36d11e4a7889380f3ec4fdcf51", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e53fb17eb164b7cbf4c1fc72ff385ca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e5fa6808c7a4cff997186b4b83f1604", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e63bf654ab347c980c81d788401c535", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e64f14372cf4caeb3dc956ea5106b9c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7e691053a19549d2822089a2de413af2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7ea1f503c0e740058149e3698cc2edbb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7ea4a6049ca7465c8e70780fce724d1d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7eb05c8b1c524f40b262bc2cb6ef4e56", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7ee891771ef04224abe47e98ffa21d96", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7eea6fe6f3de478db61733a7e9257b74", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7f231bddb9cd44a28d52c4dbdf8ba87f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7f4a217986d245b09d87a518d25b69ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7f8433d843f94d059b56825f77aae456", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7f903577fd9642ab945b0cf5aaedb0ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7fa8807e8b0949c4ac8099ba3bb48211", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/7fd72c2c91b44093a39d6ba7d1fa08d2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8013ccae00424a6bb4bf98203f0dadd9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8032d60b76d1428fa8baac54d7d961b8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/80444d0c61dd40ee9de8e52db4bf65e2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/80715fe35da94e2c989c49a9a37b6559", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8126f07506864d1a85f06a575329d4b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81312f5eebe94ee981f397ae73294549", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/819c8885b7ec4f38bbe748a1d03085e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/819ee2e239414fb1aab29c9eeb8f1cd3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81a70846106c4518a2809ff3772d04bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81aab5dfc5f544309836db85b91d22de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81c494a1a5384b47a22220dd3418ff63", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81dc6e5ca4ac42dab2371c2113275451", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81f89f51bf204aabba75dc1826985fee", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/81fc3b92aa5947acb907d8871d99577f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8209864084bc4f78b22bd17bc3e772de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8219df04815a499f97e5f0f3d797ef3f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8220092870994e139d65aff7c09a7926", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/825fc37b3890428c98c8e5b3ba5a06ca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/827822640854465ba70dcb3f99fc9bf4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8287cee809794211ab080168ca128888", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/82aa82a7081e4d1a9f318820dc20357f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/82c229f5c21446e0a9a2a8124351bd63", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/82d8336a6e0d4aa0b505db9311906b65", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/82ee0ab8ef86400ca4af2d117b72aa02", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/82ee2edb101146fda84fea12cc5f87bd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8308a459aef6434c8f026cab1195b656", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/83177fcc5cfb489aa0282dfcef05486b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8318c21efb9146b4b76241e526a300f6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/835036b46edb4fc798d7077202b3874e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8351e4614a664c09ae1c8bf6401ad196", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/83599e4f6a4142c0a125dd8a1f52b18f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/836d08125a3a422686957cdf35d2a455", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8370656863e04ac68a464e0f34fbe39e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/838b78a508134f46bf403abda942d87a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/839168757a784299a53a2cd9b4ed3aab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/841c6e652f6240daa5166429e09afa13", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/842aed6700dd41698a73c3fd3181c117", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/84310d7030354ddc82a6468a3c02244e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8467e1051da54e95aa7dcdaa41100716", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/847b75b555564a8e8210c3be0e5eb734", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/84cd5ed2da0442fb8b4600deb9158323", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/84f922542af447c59d32facafd823916", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/85297b7ba1a145018b2a2d5f0c695f67", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/853845603f6346b9acd7f64fd8f44c49", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8550ded8935f4aefbc745a5bfb05e513", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/856c1c329411461e85dd24e8fdf6ad80", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/85bd50ae9bf045288094ae165cd53dfa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/85bdfe1e275a4b59940c49dcf3a0103b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/85d76049fe05424eb8fc52cc9370c4f2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/85f5cbd479064b318aa0fd5c46edceb4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/86095fcff297481d8b4361d5458517c6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8662140044fa4f029bc1feeca19b0178", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8666c09ad10b41108ae85a19ee389626", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/868c68371caa433397c41f54008d39fd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/86cbaf53d22b4505a5d5f1079f0b0bfd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/86e650efe56f443793b5e6f300cc0985", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/870241720cac4d8ea64acdd681e3f525", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8721568fc9b9467db6928311952e2ab7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/87395a3e76aa4968a01e1f63c183bfc9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/877c423669c5491cb393b8c63267a7a6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/879d24db5bb2433b8ded073a6cef1173", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/87cb51928edd4a34b533ef201f61704e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/87e58a7867a2410eac560730e4ceb4e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/881cc59b19b14fddbee8e898bfa045d2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/881eb764f2fc499e934b0beed962bbd3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/884788e98c5643e6b1a10562c3d62783", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/888a4f545217477ca99e6cb44a60f8f4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/88b2f6e0d0084c10b1ed1a1b86596342", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/88df0be00e704badb2b87d3372e3fa75", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/894dc8b1458d4563bc0440c2fb717cf7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/89607729ddbf4ecb99f3a0db9bb9062b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/897c2bf8550a4b80a94675d14ac2afea", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/897e10a7c96a40689945e532a6da09d2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/899d941c7f5c45bb83789bc3da2fe1f7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/89d8b1968c8e48f6a21bee1c90b20530", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/89ef5711de744d188d82cddbab123240", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/89f090d974244688a5b97762fa4a9b77", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a04cfa8cd5e4b119346d617f8fb9d2c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a2354acf82144bba5298bd453291666", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a3a9b49a90c48778416c5ed64483bc7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a4c9023987849e1886a00b820895190", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a7b3148a1c94e23b2225bec8ce96729", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a7cbc0fcefe4c9aab035f9959e6c479", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8a97f9a8fc414c418850e61e7e6e27f6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8aa0e08c704f4574ac6fd0089873b719", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8aac69beff4049c893bf2362b285b0c1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8ad366c9a3114f1d9cd76b5ea3b240e0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b0c635837b94d1a8536c484a24a39df", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b1f42a7802c4417a4ee11b317888419", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b221187b2c94ab6a3cae6a9c8953cae", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b285340f248467c852533ac2dc3c3fa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b534cfdb5b84fe98554c4029d3edb2a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b7e5b0b0139414597fffb9ffb5b5b71", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8b8f50e6ffa043be9306a4ff67d06d8d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8c20aca312314a11a2e93e97b8b91bcc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8c5990d52fea4d6a8bafb0cc2f85d93a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8c61dd4f22184b75b837263453ba48e5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8ca3e1415a7f458fb12c11f187c2147a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8cb82d4a8a4749e2ae290e90698a5f5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8cbe56e90c5541a3a47d2f00603d4fc1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8ccad82b6a664a2a8d543b2783d8bd17", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8cfaeca0c08e480ba414805b6b16cf6e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8d0fd5d6e57c454785d44e8582ccc0ab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8d42995b6a054b759d9ac164bfe22977", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8d4cf98c5f4443e9aa74b789a6dd9666", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8d4ea8f950c24095929397a89b2cf19f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8d582f589dee40c3944259319e887a2c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8da64599b7874266b2592e413bd01242", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8df838e4cf124d9399c5000a2bcc304c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8dfe55bc646d4e178f9a61133322e3de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8e359e620967443d83de16d90aa12587", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8e388bbc0a854e6f897e8db792060261", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8eb500863d3d499daa6b059fc9f5167e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8ed6603ae18f4557af43bf207ec97e78", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8ed932431a4646f7b96fdffb72e81599", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8eddc7e6e94a42ebbaf5b2ce3012c9c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8eed8b2e70494ab5a1453cfa29a08d90", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8f44704c5a0c4b388b25debf46ae2ddb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8f80ca2377624771837474da9542009a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8f8d66b28a79433d86eb6b135b679e43", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/8fc7ef165b6744ab98ebe9a6fbc0f19a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/900386ee46c94c2d846bc04be874e030", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/90319f9c389e431c825a9f7caeaa46ff", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9035a5c10d32488594f3134911494f1d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/907874a810ba45879171af1e8096eff8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/90b231c2f42c4b4083d09cf045551635", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/912fa19c41fe4634876f15a21662e918", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9151d900fbc348fca47f4e1bfaeab4ef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/917c268213b54dbfaf530b8d272c429c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/917ff9458e6f42f28759a7d408a7f0be", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/91a2c8a1894049d9b041a79a6bbf1866", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/91a479748fb54f4a83f18c01914f360e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/91c8b1147e244c62bca8ef9523ed04fa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/91cf47ce4268411c8a061b653cdfd6c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9221b92e8aee42638cf3cbf706eff468", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/922b4fe577f54d7daf690864c3b44714", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9263a135f9824c9e8de3ea4d14e9b5a7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/92c5690ee56342d397a3eb9379ce3d7e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/92f75416154649788e80b2953910e8df", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/92ff35517846420ea9acce8306e5ea45", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9301c9e21d2f4099a76227624248d924", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/930435fafb55459e9ab0e93857dbca78", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/931277753d5143d68aab030e296c908f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9327c00610bf47548549b2535b789790", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9330b8d39be2461a975445396047bc0e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/935b4afb0b8d4814b39df6ee2c68aa53", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9363474d98bb4571a2eb61343a3bd05c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9389967d94dc491fa18dacd590389976", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/93a4e86b93f647049c88ffe4e8f324b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/93acd48c794245b6bec20a6f709c2c2e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/93ba922e48a1418da14da986728caef5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/93be3e3414e54f7aa2b61b83b8057580", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/93d2d1c3846840a78237d1bad3633528", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/941132612e2d49f2b34573712744a343", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9416ddcae185418e92d3876af48f5363", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/94393d9a381b411ab120c506e52c7437", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/946700465b2b451d8b661cc15f28244f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9478da7fb15b48ebb27462ab1be6be75", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/94871bd798c54845a6b9b2fca1e98e74", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/94c55f9598db42ccb1d1661168ecc31d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/94feedf21c4f4600a2f7ab7d068beb73", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/951357a57717459fbdbff1fc814ba50f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9530e698779e47309f34054ed8d4ec0e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/955b7ffcb12941468fc28f9a079752a0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9572827e2fe14655a176f0039dfd64b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/95809b974ec2481b91951ba2369555e7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/95bdca2d2447448e9b311bcddb7fe3f5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/95f54069c07a4a93b2d3af783d3ac77e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9639bce674c1497a81383f4eb7b043ce", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/96580589db5a4fccb5d03ef93cadf667", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/96bcb3c5007d4160b189b018e30b86ed", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/96ff9e716d62440abb489fbbde7de38b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/97077b62b94a4fa0a90bae28b250f1cc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/971c1a9e93ff437a8b01b63049e32d6d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9731845f5f1d41639c527db6fbf46cf8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9736a36d5b984156b278cdc677f59bc9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/97552d11ac6b457f8d9e38623d26e3bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/97742fd2573f4a7db416b1ce4152a06c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9774494f6bb54112b413f6348154aef9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9777c618506f44a49191583f98099c82", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/977a8b736c304cddba5097a86ddfe713", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/97958b4ccd784ed79e3343452ecb237c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/97fbbe35c7194b898bb2030fa234d1b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/984b8e0bb0ab4a3eaf0aae2ec20367e6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/984f2bcb50fc4d49ab3e21e43956f2f6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/987a9a6e37e7495c98efb4362dccc31e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/988913956da64d5db501ecb4f83973b7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/989a15de37ca45439ff1e2e13b14c8dc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/98b85e2d1f2a48838445a2484790b93f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/98ba78acbf5d4fdc9871d221257c30b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/98cb8a2e3f88454684008edfdab3784a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/98edf217c2e04cf29c9960d5958efc78", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9915e8f8dadd466cb1656f6510f0b86d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/992c3d37269b41df909f3837a1c291eb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9954401511e14251bb564af3f2edbc5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/996e24599218411095df73e4ef8328b4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/997cbb82dcb54cef84b63bd275f534fb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9997a790a1984a6cb67c14136b2ddd39", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/99a2a17db4ba4a7084ca9be2c8a46883", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/99d7222d77d74d99b85a78ceae86dc3a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/99d775248e82497ca4b34481719bbcaf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/99e5cc6aa1644cb5b3f3fe80a5e17374", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9a0824995de64273a49bbf7ed76f3857", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9a22ddb7b1ab45f5ab0df4b029a17eba", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9a4b9667c29c4e9ba4ac585c64052e37", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9a87a129c598413fa1ce47acc6b54b52", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9ac83f8ea1904fb9a43ed7fad86996c5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b0a532b3a914772bbd121c78ff096c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b0ef8da36b94529ad973c3c9d2042fe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b2ef9380248400191afe30ca2fd553d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b4707f56f7b4bf18ebe571d36d3483a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b549d3ab9f748aa90d6ad50aa6b36ea", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b54ef137fdf4c03803d6b4dfecb3a3e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b554488e79444978c8b2713fb76d4ef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9b870edbf277486292abec9da0817cb8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9ba097399a3544df857d1d8a213f792e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9c001f2455c646f4ac3955c7e8b406e3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9c1381e1b2bb4efa8f214273c94bd06f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9c26d3d5495049a5a7d8c70cf30e6017", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9c33fd190d7f40aebf684b351be54d0e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9cbfbe00939e4636b5ca02207084dd74", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9cc844535afe425b92c1bfa1cda82234", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d105e03e191430391c1f5a1ad681b01", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d30811081cb41e6bad1eb4ad8f70eb0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d6144433b4f42abae3d45df2e359924", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d6a1466e45548bca0bf193bfa92e091", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d79db10870a417489b068b49d64263d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d812a42fde9404ba2b4c1fa2a8380dd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d9a5fdc70624fd99c10066c9c188010", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9d9e53cb2a484e078e9324e1ef1a4790", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9dd75b348b834b8996a3f514dd4f492a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9dfd0e9adcfb4c52a87b8b565909c8b0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9e264aeec19d48c982fd290487b65004", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9e29d8a7365646e987c40315b8d19268", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9e3b5d9723c7433ca647d2bb46bf82bb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9e3c770d742f4164aaac58a11293c735", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9e4205ff374d4ad89ad9ee7e28cab43a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9e4865beaa5a4055832a3edef2d90d6a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9ebfea7a50f947f7be67514f89597364", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9eebe46918b34fd4a2ff73f703166825", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9f18f3fd911543e9ae0b3668d61109f8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9f33b246cd4b410b820606c609fa7f95", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9f52660ff14a49a89af3ec96b6949a8c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9f5fb479801549efae980a7100cf7938", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9f8c4a2d5a894f689ec7bb5affca13de", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9f962852d0414821afc7e0f87616043a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9fb68a04c5a94fbb8a730afa13956574", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9fcfce21b9d24534b1e362b0e2924214", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9fdc5aa8886547fdbc7d627899d59ca1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9fe0c801d58340baa7f920fc29a795c0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/9ff69ded22fa48fca5f7062bb6e8b399", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a00bd110650f4549bb33a17a1455a968", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a015ab5e76ca45eeb7643da66b131193", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a039c914de7a49b69b3e806066808c18", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a049c2c95edf46ceabfdc8d5dd90245d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a0552d56749d43eead39a5026d1a5632", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a0fe726331554ff58606e36b2b325638", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a11cf2fe54494513ba71d80b15f21533", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a13be9bfbf1742b780115da9f86f01e1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a148d9dfebf64d8b877b79052d75e868", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a14abb696e6f44c192461596621350c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a14d01e9ac334a48a0d29d93612962a0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a151dbd7a6994cf987ddea6e222e2d4d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a1618cb02f8c4703a25d454c2733936e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a17d9fe774c247c7af748c078d7357e0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a1a215b486704de28ea001e2899870d4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a1a84167a4b240f2943ac795210493b9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a1c6b5a6f82c4e0682b2e543e8241a90", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a203f18f1fdf44ad964199caef16d239", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a262bbcbfab547d2a6633707e7e82d38", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a26a954ad9a8416d97edff74b6f4a36b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a27824e860224ab4a34da07d117d02f3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a27edd1fecfb4bbfa5df49891932f702", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a28e32bdc5b34141bd841fd179b75527", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a29213ea060448cf85b78b962d216864", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a2960edc28284094adc54a6929afd7b3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a2b70aa9d69a4274bc547c37f53e56d8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a30290baf22749109aae1fc8f21742ab", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a31dd2e2275446abb11e7170dc7678d1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a31e33d26dcb4e779df3c24fc095006d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a3238a0b6924486bb96c0a4f9552b394", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a3267485b07a498ab5e6c64893ee73ca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a34bbbf03063470eb54ecfc9c5efc796", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a35d972e3ddc438d88946e188b41c63f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a378540b3b864594bc741a68d52fd774", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a397f988d2f1482882636c322dc09cf0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a3ecc2fa3303489d8d33fdf4697a09ad", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a3f93a907c7842e9a35caff5e0f6de1c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a422e55ad8174725829cb559fe3b698b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a4233867188640b3b13204d744637aa5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a4286701cd364d14bc80ce9e76b982db", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a432af7f2f494cebbc51713923cc8df7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a4435db8889c47eaa8aa3a8a23117d00", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a446186a61b044e281da4cb562eac5c2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a4740c2a855948818973977922660840", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a49dfe115d3249049997c52d44a67d27", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a4cd9068ea484ff1b010230e6cc99fd7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a4cf79b4501345bb8397ca7212f02e2d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a5259c7ea96643bfa89f957f1b4e14bd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a5392d92be294cc9990e2a9c716eba29", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a583d4bc7dfd41fcab56e70afcb95244", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a5997fdbb5f74b4aab181df0e6b85222", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a5bd73af83d24429b1339c7b4924efb3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a5d06134a6424485b8a86ad37a131664", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a5ea46bc1d284c41ac279ddaeed7240b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a617a9b1ae9d41f281aff255383b7b17", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a630589282df428e964698d6e1e59d33", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a64263f7d4ba4b66afb9d7a6f66adcfc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6614984736544ae960b5382b3648739", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6695fd6795744b19636607635c5a6f0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6924ae55a754201a33b11de5ea4a439", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6ac8ff7e1d345028284655eeb6bc0eb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6b481c068cc4f54bf20472e88ef214c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6caa707310f400fb7587c743a678c60", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6d1322fd833498ea58c97db18fbab03", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a6e48479cdd644658d5b3a63ebc10900", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a720a1396e2b4d568a4c17748f4529d9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a720bcb18f6d45f880d441d332a7f03b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a72bf525990846de9678cfa480f9eb71", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a730d47c70864b9f955f90cfb48b5254", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a764861e6ce04cc6a64214686717787a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a7679be513d14c86b76f9c24c1bc58cd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a7881b65ff5b469aa0a05c8fd51d708d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a78a7b8790d04fff9d247985357a2c5f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a7ad85963d3541e995d9555ad8a8d5cd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a7b2b6a4418841e48d5958a77fcd509f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a7caedb7540a4b348e7e015e8590cf63", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a8331dbf969d44798c7a32668403558d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a84b3667874044068028cf99d082f8f7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a8742955c1784242a3720a8ab90d6394", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a895d9d087274c81896f2d7b2c011629", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a8e3a9b41b1c4efbbc9ba1bcd7cb4f5f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a8ef656bca304c65a4d8c5fa427cec57", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a8f3065a2fce4f9a80ad5f8fd9c3a337", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a8fdac39364a445a82df238dab7d96fe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a94e91cdce684902a3e7f80bf81fd291", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a94ffddeefaf4063b7de2238fcdb5c95", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a95049bdf3c1498bbccb2f8c8dd3b26f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a96d461e2773467d9a1a273bb33a3c9d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/a9d877d9d753445b848d1330b18a63e7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aa51076381c3466fa984ad6893d956d1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aac8e24204e342e9942a82fe9497f062", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aaeb929c91754f5ab6aa7fcadc6153cb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ab289fc3607846619565de9363d76dcc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ab3dfce6a59345d9a50f882fdf4491a8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ab772d65e61a4115a7a29b36261b46a0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aba917f6775941e6b3e2572ae775ec16", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/abd9975543aa420ab10a449342147da9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ac0e7bc5e2e6403e964d4a3d28989b71", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ac3461f507d946dea70e0110aacb911b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ac99e3f5fff5461d8489260c86b36df0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ac9f99e01af9438aae76af0029d6ce3f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aced4063112d44ccbced1155d6018f26", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/acfc7f7f3b9f44c8a63a592c68875b5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ad2e0611cb014fe99fce3c9bf44587f3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ad8094922dc44e6fba55ca10e8f8d286", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ad960a4d42ad46ea9373933a611f5b8d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/adb366b987f3407197c7abf2dfcc1926", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/adc73956b4854ad7aac1547621676ec7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/adce24aacad14853a87d436d8faa501c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/add58657a40b4cdbb2c3bd79bb52fb17", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/adf7574fe90947d3a9166eedcc2d4d26", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ae0b05c6ba5147f694ad4939019f61fe", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ae2d362a115a4c8286fc6f79d92719fb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ae326788004146fd95b1f8226c4f793c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ae96de0a5c58445ba5186e9d16892d42", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aee3eec604044b09a04db15e59fe9979", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aeea41fae3d54930a7b7a177d4b360d4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/aef7275e39f7443caf6bd92e74dd8ee7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af04eba569b14d8d85954498ca5bf5a3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af1352bb4ca0435ba37b7df3b380d446", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af17a2b72178487faf74ed404602e611", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af3aea8128e944b3bac58a33576ff3f4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af471eb880464132ae7b6c77165bc1c5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af59c00dc40541669aaafd6399acc4eb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af86e48a9fb745efa29eec662b7f486f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/af8c4d1b53934bac8ef5923f4b118257", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/afca0cffe1534451b54b337be2ce4b80", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/afe0d198d5f94610bcae42f8c5a44a80", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b0113c55395443ec8cfc879e0d364000", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b037e5c6663c47839ad9908c07cbd75f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b0570af0ec094514990b3802a63252eb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b06bbc53a57f401c9a6ff11ef9cf93e0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b07494c1d38a4480b4be0d91be5b4f56", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b080d2bb8bc94a97b74be66de49b95c8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b0988007c7cf492080f6cf3a610b3260", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b0c0d8d3aaed480f89843b0f64e1641c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b0caaae131d049cd8e4c264ace0880d1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b0eab433a40d4032ad9e9563e6aff16e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b104befdacbd4be1b42417f85f3eb58d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b14d7263e9624d058ee603e681de5cb6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b163224bbcc847118dc18c8ba98284f5", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1712610952b4d749a5a28dbb7131b8c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b179def8850c49858bf6914f853d9e71", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b18847906cfc4bdd8b90328e1235a228", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b19a840af2eb40ed98ff332b25e0ccbd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1b53c20ff854d839f81f6664efd9969", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1c7b8704ddc4e758ff34de0b56d5115", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1d45226951041039b02088121cab97b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1f44679fb084a2191faee8df0d56d5d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1fa1902165e4ab8ad446b758387c8aa", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b1fea011b852441d96287aee26d3e086", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b20784c3cc744614beb3dc13b6b04ec2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b23ca572d60448eebea6556860107023", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b23d6bd51b4046d19bae402af4578401", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b23e815028bb40b2833adfe094a279c7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b246d9484ed94ee4bf012e365589f5f0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b282c41e1283430d8fba0ff8be80e234", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b28de733bc8d478785bccc825f12d5a8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b2ac7878873b4d7eac9fd050c526c367", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b2b25213f4eb4452aaf2ca8ddca56105", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b2bf9a54b25e4df58fb5b2be65837ed9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b2c94e90593d49e694ccdc55547e23fd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b2dfc9b19bfa454e847e7902c0ff1c7a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b2efc51827fe4b4cb9ac4b22f6d03a0b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b310b3ba81d24be0927f69456f03dada", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b34335c688db48949455492630c51a2b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b357e31e730f48e69db65fc42bbd6a13", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b36b1b0312b542429c5cf1e371d09bca", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b3754fcbf8a045babe02f698c37f63db", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b37dc76cb8de471d8da3d0713e481171", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b380eefdca214473992a978c9bacbc35", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b39db56526c341c2aa5701aa373143d7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b3a3104fedab45f8887a4effe5628e19", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b3b0fad234eb40488f677d636d8f3e8e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b3ba6452cc18469ebd3b621e40691f1a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b401b9ddb0b74fabb11737c641ede71d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b415f61b971b4417bfa7bad3c7906b7c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b41a670fd20443d4a0be9d99167f2223", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b46f4f32afa441a6ad6b97ff12ace769", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b47155855d4b499f94451670b7be8c1a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b477d4311ec34373becf6c9f6a8f2d63", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b4ce44b9099f450b999f20c4b15d536b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b4edd2328ab34fcda6fe5f03de281db9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b4f20617deff44c4933ff40f76a63c81", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b4fb115c305146ebae4bfe4f37b59534", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b50595237fdf48949fcb519d3cd4e2e9", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b508acc1d7dd44e683efbe45675abe20", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b50a73501b55494dbc9a9b439a5c2bfc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5244fc71b2c4adfbc87f1e7ea7b794f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b532aef8b4794680b1f8db0ba87e4725", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b554b90f24474717ab8b31591b8072f3", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b555f8a9282541cca651208ee6b571c4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b55bfc776035498fa4ba27b852ae0f2e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b560515c7b3d441291125246f6d68e4e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5665621cf294216b88678b1bc32fa4e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b59073a83549455489b9a2882f69687e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b59cd6c9e1e54ce8983285d3fd6e97c7", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5a1108b1e7d40a997d004b4348213bf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5af40da81284db48b839b3569a934d8", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5c6e49c16f24eeda845457a52a4d68a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5cdf0b02ad54098b79303f9f142df22", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5d4c6d881474c7f894a3aea7cc0911a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b5e832112e944f908fc393fe65a9134b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b636671af77144fd8f4425ea5c3bdc28", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b6533b975c1d43b89d39552d840eb61a", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b6880f45267948079d9ee6bc7c4d2578", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b68ea28f32f149de944091d653e45c12", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b69991a1c16a4a93acb45ee97911c3c2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b69b701843a948c79a6bb39080aa4131", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b6c80feb50fa4b9187e1565ff0e084b2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b6e81d5b30be4501924388bb47b98143", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b6fc3a2e36e746d1950dfb52ba9a3cb2", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b71c64feaaee4ad198518431c1ff6fd6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b72d02ad4f0544efa7f972ce4d3d3220", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b74705218528446ea274eacc29998452", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b75b1c00a0ba4706aa6cf6e000396bcc", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b75c4ae4b525476784700f2ddd4b9456", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b76e5886058447c19518edd783fdb5bf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b76f2b9a0e824c2383c4d3ec6ef1501f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b7749d5f11964492acd4b9c3ccdd7760", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b78f19ec926142bc91856aa0718d86bd", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b795a9c1cd4a4e9e9bd1f0c91d2dfb69", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b7c89a91533b40508650a40cacb0393b", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b81d959eef114f4b86e159f93b46079e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b841a1ac400243a79864a061f7ad0865", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b88457f0c4124eaea0e51023c71e8a2d", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b891a3d33ee547bd8771f877abc611ef", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b896797469714c0eb0fd7af990928ec1", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b8a90fba8b374ea1b01e8c1cbdbf0edb", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b908756b8b7444ada3598a045b852226", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b914087011e947e7a7cebe4aad7494f0", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b92e36ac11d74c55a6a3ea61c8376a84", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b94e634c07894fcab12de29fd55042ea", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b9626ac591f54565ba147f1a725475da", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b979635be97643dcb1dc69040a434365", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b987fb0b54104613b06c1912329d25d6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b991e3e373164eaaa21090ec63c83549", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b9b46a2fe038413d9c53e42757851d17", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b9b68b73617f4e6c9f0fd734d4fc9651", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b9c86fdd302b48cc860c5a1553f27944", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/b9cf90fee7744bfdaf7420f4c8eb76bf", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ba09293f597a434f882d6a0e2fecbf0c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ba20b97d6b32453c88d287a051f70569", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/ba56b4fe89e54fdaa7f0b31836b9a8d6", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bac9769935d740f581c79b18d68c85c4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bad9f5fe3607488891ef5ec45b017c59", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bada59db8c0a45158c8e9830f9595e77", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bafaf97a4bb74dbd8081a333191ed30c", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bb135986699b4e93bd4a1fab0663ba3e", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bb40aad35c0c48b0981f7d49dbae1621", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bb49ead8af4a41a0a23b184ab3bd3bd4", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bb5a131ac85c4205b198626b17a0a630", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bb6139155d7d4f8cba1e9ad9e7d37c28", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bbbcf5534d884c8391bfd649240fed25", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bbc0120cc6da408ebeb4b537d28de320", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bbc49b40992745e78ceff5c3703c4e1f", + "attachment_id": null, + "used": false, + "result": "applied" + }, + { + "key": "o/bbcddfe0e4094f90b606816b0ef99fc4", + "attachment_id": null, + "used": false, + "result": "applied" + } + ], + "restored_object": [ + { + "key": "o/77f4fbc630a1966cb3f6d4e09c019318", + "attachment_id": "ed0ff4c7-507b-44cd-8928-622ea6e977a8", + "used": false, + "result": "applied" + }, + { + "key": "o/ee7f3d21bebb4a56572b9b1f7d2299ee", + "attachment_id": "4c72dcfe-c46f-47a0-a442-2937c722ed99", + "used": false, + "result": "applied" + } + ] + } +} \ No newline at end of file diff --git a/evidence/reconcile/interrupted-3-third.json b/evidence/reconcile/interrupted-3-third.json new file mode 100644 index 00000000..673e62d1 --- /dev/null +++ b/evidence/reconcile/interrupted-3-third.json @@ -0,0 +1,38 @@ +{ + "mode": "apply", + "grace_seconds": 30.0, + "verify_checksums": false, + "duration_seconds": 0.103, + "error": null, + "scanned": { + "objects": 186, + "rows": 186, + "consistent": 186 + }, + "findings": { + "orphan_object": 0, + "stale_intent": 0, + "missing_object": 0, + "size_mismatch": 0, + "checksum_mismatch": 0, + "restored_object": 0, + "fresh_orphan_object": 0, + "fresh_intent": 0, + "fresh_missing_object": 0 + }, + "used_attachments_in_findings": 0, + "actions": { + "planned": 0, + "applied": 0, + "skipped": 0, + "failed": 0 + }, + "cost": { + "s3_list_requests": 1, + "s3_head_requests": 0, + "s3_get_bytes": 0, + "db_pages": 1, + "page_size": 1000 + }, + "samples": {} +} \ No newline at end of file diff --git a/evidence/student-tests-first-run-f9f5ab9.txt b/evidence/student-tests-first-run-f9f5ab9.txt new file mode 100644 index 00000000..360812b7 --- /dev/null +++ b/evidence/student-tests-first-run-f9f5ab9.txt @@ -0,0 +1,414 @@ +# Own tests: unit + stand (pytest -v); failures restored in finally +# 2026-10-25T11:43:04Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов + +$ python -m pytest student-tests -v +============================= test session starts ============================= +platform win32 -- Python 3.14.6, pytest-9.1.1, pluggy-1.7.0 -- C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe +rootdir: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean +plugins: anyio-4.15.1 +collecting ... collected 320 items + +student-tests/test_api.py::test_register_never_returns_password_and_login_returns_bearer PASSED [ 0%] +student-tests/test_api.py::test_duplicate_username_is_409 PASSED [ 0%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[short-11ch] PASSED [ 0%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[ ] PASSED [ 1%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[ \t \n \t \n \t \n ] PASSED [ 1%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 1%] +student-tests/test_api.py::test_edge_passwords_register_and_login[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 2%] +student-tests/test_api.py::test_edge_passwords_register_and_login[with:colons:inside] PASSED [ 2%] +student-tests/test_api.py::test_edge_passwords_register_and_login[Twelve chars] PASSED [ 2%] +student-tests/test_api.py::test_wrong_password_and_unknown_user_give_the_same_401 PASSED [ 3%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic \xe9] PASSED [ 3%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic !!!] PASSED [ 3%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic bm9fY29sb24=] PASSED [ 4%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic //46eA==] PASSED [ 4%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 4%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[] PASSED [ 5%] +student-tests/test_api.py::test_logout_revokes_only_this_token PASSED [ 5%] +student-tests/test_api.py::test_x_user_id_basic_and_garbage_do_not_authenticate PASSED [ 5%] +student-tests/test_api.py::test_malformed_json_is_400[{] PASSED [ 5%] +student-tests/test_api.py::test_malformed_json_is_400[] PASSED [ 6%] +student-tests/test_api.py::test_malformed_json_is_400[[]] PASSED [ 6%] +student-tests/test_api.py::test_malformed_json_is_400[null] PASSED [ 6%] +student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}] PASSED [ 7%] +student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}] PASSED [ 7%] +student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "\ud800", "password": "Twelve-chars"}] PASSED [ 7%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body0] PASSED [ 8%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body1] PASSED [ 8%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body2] PASSED [ 8%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body3] PASSED [ 9%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body4] PASSED [ 9%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body5] PASSED [ 9%] +student-tests/test_api.py::test_unicode_limits_are_code_points_and_nul_is_stored PASSED [ 10%] +student-tests/test_api.py::test_oversized_body_and_wrong_content_type PASSED [ 10%] +student-tests/test_api.py::test_chat_validation PASSED [ 10%] +student-tests/test_api.py::test_foreign_chat_is_indistinguishable_from_missing PASSED [ 10%] +student-tests/test_api.py::test_chat_list_order_pagination_and_isolation PASSED [ 11%] +student-tests/test_api.py::test_history_pages_without_loss_or_duplicates PASSED [ 11%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=0] PASSED [ 11%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=101] PASSED [ 12%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=abc] PASSED [ 12%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=1&limit=2] PASSED [ 12%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=] PASSED [ 13%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=garbage] PASSED [ 13%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDF8Mg] PASSED [ 13%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDk5OTk5OTk5OTk5OTk5OTk5OTk5fHk=] PASSED [ 14%] +student-tests/test_api.py::test_cursor_bound_to_chat PASSED [ 14%] +student-tests/test_api.py::test_read_after_write PASSED [ 14%] +student-tests/test_api.py::test_malformed_cursor_values_are_400_on_both_lists PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[\xff] PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"first","TEXT":"second"}] PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"a","text":"b"}] PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":123}] PASSED [ 16%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}}] PASSED [ 16%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}]] PASSED [ 16%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x"} {}] PASSED [ 17%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}] PASSED [ 17%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":null}] PASSED [ 17%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":["nope"]}] PASSED [ 18%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":" \n\t "}] PASSED [ 18%] +student-tests/test_api.py::test_ff_byte_body_on_registration_is_contract_error PASSED [ 18%] +student-tests/test_api.py::test_text_is_stored_verbatim_and_length_counts_raw_text PASSED [ 19%] +student-tests/test_api.py::test_unknown_attachment_ids_are_404 PASSED [ 19%] +student-tests/test_api.py::test_member_ids_are_compared_as_uuids_not_strings PASSED [ 19%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer .] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer a b] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[bearer\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 21%] +student-tests/test_api.py::test_login_with_a_body_is_rejected PASSED [ 21%] +student-tests/test_api.py::test_password_range_is_12_to_128_code_points PASSED [ 21%] +student-tests/test_api.py::test_every_api_response_carries_instance_and_request_id PASSED [ 22%] +student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\xa0] PASSED [ 22%] +student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\u2003\u2003] PASSED [ 22%] +student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[ \xa0\t\u3000] PASSED [ 23%] +student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":"x","password":1e1000}] PASSED [ 23%] +student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"ff_probe","display_name":"x","password":"12345678901\xff"}] PASSED [ 23%] +student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}] PASSED [ 24%] +student-tests/test_api.py::test_more_than_100_members_is_400 PASSED [ 24%] +student-tests/test_api.py::test_cursor_of_one_list_is_rejected_by_the_other PASSED [ 24%] +student-tests/test_attachments.py::test_png_and_jpeg_roundtrip_byte_for_byte PASSED [ 25%] +student-tests/test_attachments.py::test_metadata_reveals_no_storage_details PASSED [ 25%] +student-tests/test_attachments.py::test_original_is_not_stored_on_api_containers PASSED [ 25%] +student-tests/test_attachments.py::test_outsider_and_unknown_get_404 PASSED [ 25%] +student-tests/test_attachments.py::test_attachment_ids_are_checked_per_chat PASSED [ 26%] +student-tests/test_attachments.py::test_message_with_attachments_and_history PASSED [ 26%] +student-tests/test_attachments.py::test_idempotent_replay_with_attachments PASSED [ 26%] +student-tests/test_attachments.py::test_message_body_validation[{"attachment_ids":["%(a)s"]}] PASSED [ 27%] +student-tests/test_attachments.py::test_message_body_validation[{"text":null,"attachment_ids":["%(a)s"]}] PASSED [ 27%] +student-tests/test_attachments.py::test_message_body_validation[{"text":1,"attachment_ids":["%(a)s"]}] PASSED [ 27%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":null}] PASSED [ 28%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":"%(a)s"}] PASSED [ 28%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":{}}] PASSED [ 28%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s","%] PASSED [ 29%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s",1]] PASSED [ 29%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["not-a-uui] PASSED [ 29%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"","attachment_ids":[]}] PASSED [ 30%] +student-tests/test_attachments.py::test_message_body_validation[{"text":" "}] PASSED [ 30%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":[],"sender_] PASSED [ 30%] +student-tests/test_attachments.py::test_at_most_ten_attachments PASSED [ 30%] +student-tests/test_attachments.py::test_size_limit_through_the_load_balancer PASSED [ 31%] +student-tests/test_attachments.py::test_pixel_limit_before_decoding PASSED [ 31%] +student-tests/test_attachments.py::test_exactly_40_megapixels_is_accepted PASSED [ 31%] +student-tests/test_attachments.py::test_bad_content_is_415[png: first 33 bytes of the fixture] PASSED [ 32%] +student-tests/test_attachments.py::test_bad_content_is_415[png: truncated] PASSED [ 32%] +student-tests/test_attachments.py::test_bad_content_is_415[png: corrupt IDAT, valid CRC] PASSED [ 32%] +student-tests/test_attachments.py::test_bad_content_is_415[png: fewer rows than IHDR] PASSED [ 33%] +student-tests/test_attachments.py::test_bad_content_is_415[png: flipped byte] PASSED [ 33%] +student-tests/test_attachments.py::test_bad_content_is_415[png: bytes after IEND] PASSED [ 33%] +student-tests/test_attachments.py::test_bad_content_is_415[jpeg: no scan data after SOS] PASSED [ 34%] +student-tests/test_attachments.py::test_bad_content_is_415[jpeg: truncated] PASSED [ 34%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: png bytes declared jpeg] PASSED [ 34%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: jpeg bytes declared png] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: gif declared png] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: html declared png] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[type: text/plain] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[type: octet-stream] PASSED [ 36%] +student-tests/test_attachments.py::test_bad_content_is_415[type: multipart] PASSED [ 36%] +student-tests/test_attachments.py::test_bad_content_is_415[type: missing] PASSED [ 36%] +student-tests/test_attachments.py::test_empty_body_is_400 PASSED [ 37%] +student-tests/test_attachments.py::test_bad_filename_is_400[missing] PASSED [ 37%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=] PASSED [ 37%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=%20%20%20] PASSED [ 38%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=.] PASSED [ 38%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=..] PASSED [ 38%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%2Fb.png] PASSED [ 39%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%5Cb.png] PASSED [ 39%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%00b.png] PASSED [ 39%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=c1%C2%85.png] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=tab%09.png] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=photo%E2%80%AEgnp.exe] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=%FF%FE.png] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a.png&filename=b.png] PASSED [ 41%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 41%] +student-tests/test_attachments.py::test_unicode_filename_is_kept_and_safely_exposed PASSED [ 41%] +student-tests/test_attachments.py::test_variant_parameter PASSED [ 42%] +student-tests/test_balancing.py::test_one_session_is_served_by_every_real_replica PASSED [ 42%] +student-tests/test_balancing.py::test_write_on_one_replica_is_read_on_the_other PASSED [ 42%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-pg_hba] PASSED [ 43%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-network] PASSED [ 43%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-pg_hba] PASSED [ 43%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-network] PASSED [ 44%] +student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[0] PASSED [ 44%] +student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[1] PASSED [ 44%] +student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[0] PASSED [ 45%] +student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[1] PASSED [ 45%] +student-tests/test_balancing.py::test_graceful_shutdown_order_in_logs PASSED [ 45%] +student-tests/test_balancing.py::test_full_restart_lb_rediscovers_all_replicas PASSED [ 45%] +student-tests/test_balancing.py::test_scale_up_and_down_is_picked_up_without_errors PASSED [ 46%] +student-tests/test_balancing.py::test_compose_policy_limits_ports_secrets_images PASSED [ 46%] +student-tests/test_balancing.py::test_running_containers_have_limits_and_no_bypass_ports PASSED [ 46%] +student-tests/test_balancing.py::test_migrations_ran_once_by_the_migrate_service PASSED [ 47%] +student-tests/test_consistency.py::test_interrupted_and_stalled_uploads_leave_nothing PASSED [ 47%] +student-tests/test_consistency.py::test_crash_between_s3_put_and_commit_is_cleaned_by_janitor PASSED [ 47%] +student-tests/test_consistency.py::test_lost_answer_after_commit_never_deletes_the_object PASSED [ 48%] +student-tests/test_consistency.py::test_fresh_pending_upload_is_not_touched_by_janitor PASSED [ 48%] +student-tests/test_db.py::test_registration_race_exactly_one_201_and_seven_409 PASSED [ 48%] +student-tests/test_db.py::test_create_chat_rolls_back_after_the_chat_row_was_inserted PASSED [ 49%] +student-tests/test_db.py::test_migrations_rerun_is_a_noop PASSED [ 49%] +student-tests/test_db.py::test_100_messages_from_10_clients_keep_order_and_cursor_walk PASSED [ 49%] +student-tests/test_db.py::test_secrets_are_not_stored_in_plain_text PASSED [ 50%] +student-tests/test_explain.py::test_late_page_uses_index_range_not_filter PASSED [ 50%] +student-tests/test_failures.py::test_valkey_failure[stop] PASSED [ 50%] +student-tests/test_failures.py::test_valkey_failure[pause] PASSED [ 50%] +student-tests/test_failures.py::test_postgres_failure[stop] PASSED [ 51%] +student-tests/test_failures.py::test_postgres_failure[pause] PASSED [ 51%] +student-tests/test_failures.py::test_locked_table_gives_bounded_503 PASSED [ 51%] +student-tests/test_failures.py::test_logout_during_valkey_outage_is_never_lost PASSED [ 52%] +student-tests/test_idempotency.py::test_replay_returns_the_same_201_and_message_from_every_replica PASSED [ 52%] +student-tests/test_idempotency.py::test_other_body_with_the_same_key_is_409_and_keeps_the_original PASSED [ 52%] +student-tests/test_idempotency.py::test_key_order_and_missing_attachment_ids_are_the_same_body PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[] PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[\u043a\u043b\u044e\u0447] PASSED [ 54%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a b] PASSED [ 54%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a/b] PASSED [ 54%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a,b] PASSED [ 55%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a\tb] PASSED [ 55%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[*] PASSED [ 55%] +student-tests/test_idempotency.py::test_repeated_key_header_is_400 PASSED [ 55%] +student-tests/test_idempotency.py::test_longest_key_and_every_allowed_character PASSED [ 56%] +student-tests/test_idempotency.py::test_scope_is_user_plus_chat_plus_key PASSED [ 56%] +student-tests/test_idempotency.py::test_outsider_and_revoked_session_never_see_the_stored_result PASSED [ 56%] +student-tests/test_idempotency.py::test_concurrent_same_key_through_different_replicas_creates_exactly_one PASSED [ 57%] +student-tests/test_idempotency.py::test_concurrent_same_key_different_bodies_one_wins PASSED [ 57%] +student-tests/test_idempotency.py::test_failure_after_the_key_insert_rolls_everything_back PASSED [ 57%] +student-tests/test_idempotency.py::test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message PASSED [ 58%] +student-tests/test_idempotency.py::test_lock_timeout_is_a_transient_503_that_is_not_recorded PASSED [ 58%] +student-tests/test_idempotency.py::test_replay_after_the_serving_replica_is_stopped PASSED [ 58%] +student-tests/test_reconcile.py::test_detect_and_safely_fix_faults_on_both_sides PASSED [ 59%] +student-tests/test_reconcile.py::test_checksum_mode_finds_same_size_corruption PASSED [ 59%] +student-tests/test_reconcile.py::test_safety_valve_refuses_mass_deletion PASSED [ 59%] +student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[s3] PASSED [ 60%] +student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[postgres] PASSED [ 60%] +student-tests/test_reconcile.py::test_live_uploads_are_never_hurt_by_concurrent_apply PASSED [ 60%] +student-tests/test_sessions.py::test_short_ttl_expires[valkey] PASSED [ 60%] +student-tests/test_sessions.py::test_short_ttl_expires[postgres] PASSED [ 61%] +student-tests/test_sessions.py::test_restart_does_not_extend_ttl FAILED [ 61%] +student-tests/test_sessions.py::test_postgres_session_store PASSED [ 61%] +student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop] FAILED [ 62%] +student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[pause] PASSED [ 62%] +student-tests/test_storage_failures.py::test_upload_survives_failover_of_the_serving_replica PASSED [ 62%] +student-tests/test_storage_failures.py::test_bucket_is_private_outside_the_api PASSED [ 63%] +student-tests/test_storage_failures.py::test_s3_restart_keeps_objects PASSED [ 63%] +student-tests/unit/test_images.py::test_public_fixture_is_accepted PASSED [ 63%] +student-tests/unit/test_images.py::test_valid_png_kinds[rgba] PASSED [ 64%] +student-tests/unit/test_images.py::test_valid_png_kinds[palette] PASSED [ 64%] +student-tests/unit/test_images.py::test_valid_png_kinds[1bit] PASSED [ 64%] +student-tests/unit/test_images.py::test_valid_png_kinds[gray] PASSED [ 65%] +student-tests/unit/test_images.py::test_valid_png_kinds[noise] PASSED [ 65%] +student-tests/unit/test_images.py::test_valid_png_kinds[1x1] PASSED [ 65%] +student-tests/unit/test_images.py::test_interlaced_png_is_accepted PASSED [ 65%] +student-tests/unit/test_images.py::test_broken_png_is_415[first 33 bytes of the fixture] PASSED [ 66%] +student-tests/unit/test_images.py::test_broken_png_is_415[truncated (no IEND)] PASSED [ 66%] +student-tests/unit/test_images.py::test_broken_png_is_415[truncated in IDAT] PASSED [ 66%] +student-tests/unit/test_images.py::test_broken_png_is_415[flipped byte -> CRC error] PASSED [ 67%] +student-tests/unit/test_images.py::test_broken_png_is_415[garbage IDAT with valid CRC] PASSED [ 67%] +student-tests/unit/test_images.py::test_broken_png_is_415[fewer rows than IHDR] PASSED [ 67%] +student-tests/unit/test_images.py::test_broken_png_is_415[bytes after IEND (polyglot)] PASSED [ 68%] +student-tests/unit/test_images.py::test_broken_png_is_415[text] PASSED [ 68%] +student-tests/unit/test_images.py::test_broken_png_is_415[gif] PASSED [ 68%] +student-tests/unit/test_images.py::test_broken_png_is_415[two bytes of a signature] PASSED [ 69%] +student-tests/unit/test_images.py::test_pixel_limit_is_checked_from_the_header PASSED [ 69%] +student-tests/unit/test_images.py::test_png_verification_is_streaming_and_fast PASSED [ 69%] +student-tests/unit/test_images.py::test_jpeg_header_is_parsed_incrementally PASSED [ 70%] +student-tests/unit/test_images.py::test_valid_jpeg[base] SKIPPED (dj...) [ 70%] +student-tests/unit/test_images.py::test_valid_jpeg[prog] SKIPPED (dj...) [ 70%] +student-tests/unit/test_images.py::test_broken_jpeg_is_415[no scan data after SOS-] SKIPPED [ 70%] +student-tests/unit/test_images.py::test_broken_jpeg_is_415[truncated-] SKIPPED [ 71%] +student-tests/unit/test_images.py::test_good_filenames[photo.png] PASSED [ 71%] +student-tests/unit/test_images.py::test_good_filenames[\u0444\u043e\u0442\u043e \u043e\u0442\u043f\u0443\u0441\u043a\u0430.jpg] PASSED [ 71%] +student-tests/unit/test_images.py::test_good_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 72%] +student-tests/unit/test_images.py::test_good_filenames[emoji \U0001f468\u200d\U0001f469\u200d\U0001f467.png] PASSED [ 72%] +student-tests/unit/test_images.py::test_good_filenames[ x ] PASSED [ 72%] +student-tests/unit/test_images.py::test_good_filenames[..x] PASSED [ 73%] +student-tests/unit/test_images.py::test_bad_filenames[] PASSED [ 73%] +student-tests/unit/test_images.py::test_bad_filenames[ ] PASSED [ 73%] +student-tests/unit/test_images.py::test_bad_filenames[\u3000] PASSED [ 74%] +student-tests/unit/test_images.py::test_bad_filenames[.] PASSED [ 74%] +student-tests/unit/test_images.py::test_bad_filenames[..] PASSED [ 74%] +student-tests/unit/test_images.py::test_bad_filenames[a/b.png] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[a\\b.png] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[a\x00b] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[c1\x85.png] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[tab\t.png] PASSED [ 76%] +student-tests/unit/test_images.py::test_bad_filenames[del\x7f.png] PASSED [ 76%] +student-tests/unit/test_images.py::test_bad_filenames[x\u2028y] PASSED [ 76%] +student-tests/unit/test_images.py::test_bad_filenames[photo\u202egnp.exe] PASSED [ 77%] +student-tests/unit/test_images.py::test_bad_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 77%] +student-tests/unit/test_images.py::test_filename_must_be_single_and_present PASSED [ 77%] +student-tests/unit/test_images.py::test_query_must_be_valid_utf8 PASSED [ 78%] +student-tests/unit/test_unit.py::test_valid_passwords[Twelve-chars] PASSED [ 78%] +student-tests/unit/test_unit.py::test_valid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 78%] +student-tests/unit/test_unit.py::test_valid_passwords[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u044f\u043c\u0438] PASSED [ 79%] +student-tests/unit/test_unit.py::test_valid_passwords[\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600] PASSED [ 79%] +student-tests/unit/test_unit.py::test_valid_passwords[a b c d e f g] PASSED [ 79%] +student-tests/unit/test_unit.py::test_invalid_passwords[short-11ch] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[ ] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800] PASSED [ 81%] +student-tests/unit/test_unit.py::test_invalid_passwords[None] PASSED [ 81%] +student-tests/unit/test_unit.py::test_invalid_passwords[123456789012] PASSED [ 81%] +student-tests/unit/test_unit.py::test_text_may_contain_nul_but_not_lone_surrogates PASSED [ 82%] +student-tests/unit/test_unit.py::test_cursor_roundtrip_and_binding PASSED [ 82%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|253402300800000000|11111111-1111-4111-8111-111111111111] PASSED [ 82%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|9999999999999999999|11111111-1111-4111-8111-111111111111] PASSED [ 83%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|-1|11111111-1111-4111-8111-111111111111] PASSED [ 83%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|not-a-uuid] PASSED [ 83%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-11111111111G] PASSED [ 84%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v2|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-111111111111] PASSED [ 84%] +student-tests/unit/test_unit.py::test_time_helpers_are_exact PASSED [ 84%] +student-tests/unit/test_unit.py::test_basic_is_parsed_and_password_may_contain_colons PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[None] PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[] PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic] PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic \xe9] PASSED [ 86%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic !!!!] PASSED [ 86%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic bm8tY29sb24=] PASSED [ 86%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic //46cHc=] PASSED [ 87%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic OnBhc3N3b3Jk] PASSED [ 87%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic YWxpY2U6] PASSED [ 87%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic 0JDQu9C40YHQsDpwdw==] PASSED [ 88%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 88%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA] PASSED [ 88%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[None] PASSED [ 89%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[] PASSED [ 89%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer] PASSED [ 89%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer short] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Basic YWxpY2U6cGFzc3dvcmQtMTIz] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 91%] +student-tests/unit/test_unit.py::test_bearer_ok PASSED [ 91%] +student-tests/unit/test_unit.py::test_argon2id_hash_and_verify PASSED [ 91%] +student-tests/unit/test_unit.py::test_s3_settings_are_validated_and_contract_limits_cannot_be_raised PASSED [ 92%] +student-tests/unit/test_unit.py::test_settings_read_secret_files_and_contract_names PASSED [ 92%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[a] PASSED [ 92%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 93%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[A-Za-z0-9._:-] PASSED [ 93%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[2026-10-16T10:00:00.1] PASSED [ 93%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[None] PASSED [ 94%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[] PASSED [ 94%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 94%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\u043a\u043b\u044e\u0447] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a b] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a/b] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a,b] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\tb] PASSED [ 96%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\x00b] PASSED [ 96%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[*] PASSED [ 96%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\xe9] PASSED [ 97%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"a","text":"b"}] PASSED [ 97%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"a":{"x":1,"x":2}}] PASSED [ 97%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[\xff] PASSED [ 98%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"x"} {}] PASSED [ 98%] +student-tests/unit/test_unit.py::test_attachment_ids_absent_equals_empty_and_is_validated PASSED [ 98%] +student-tests/unit/test_unit.py::test_request_fingerprint_ignores_key_order_and_default_attachments PASSED [ 99%] +student-tests/unit/test_unit.py::test_instance_id_defaults_to_hostname_and_is_validated PASSED [ 99%] +student-tests/unit/test_unit.py::test_idempotency_ttl_cannot_be_below_24_hours PASSED [ 99%] +student-tests/unit/test_unit.py::test_message_write_path_setting PASSED [100%] + +================================== FAILURES =================================== +______________________ test_restart_does_not_extend_ttl _______________________ + +api = + + def test_restart_does_not_extend_ttl(api): + with api_with_env(SESSION_TTL_SECONDS="12"): + actor = api.register("ttlrestart") + session = login_session(api, actor) + restart_services("valkey", "api") +> assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it" +E AssertionError: restart took longer than the TTL; increase it +E assert -0.024317 > 0 +E + where -0.024317 = seconds_left('2026-10-25T12:00:02.431630Z') + +student-tests\test_sessions.py:45: AssertionError +_____________ test_s3_outage_is_503_and_text_keeps_working[stop] ______________ + +api = +pair = ({'id': '4b97c381-df7f-48f6-92df-bb7a17232630', 'user': {'id': '4b97c381-df7f-48f6-92df-bb7a17232630', 'username': 'al...81-df7f-48f6-92df-bb7a17232630', '1a1510cc-7f0b-423f-8645-70e2fa838cb2'], 'created_at': '2026-10-25T12:01:26.054625Z'}) +how = 'stop' + + @pytest.mark.parametrize("how", ["stop", "pause"]) + def test_s3_outage_is_503_and_text_keeps_working(api, pair, how): + """S3 down (stopped: connection refused; paused: no answer at all): + * readiness is 503 on every replica (and through the LB), quickly; + * uploads/downloads are 503 `unavailable` within the S3 deadline, never 500; + * the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core); + * after recovery the earlier file is intact.""" + alice, bob, chat = pair + att = api.upload(alice, chat, SAMPLE).json() + with service_down("s3", how): + deadline = time.monotonic() + 15 + while api.client.get("/health/ready").status_code != 503: + assert time.monotonic() < deadline, "readiness did not turn 503" + time.sleep(0.5) + for replica in running_replicas(): + assert probe_inside(replica, "/health/ready") == 503 + assert probe_inside(replica, "/health/ready/core") == 200 + started = time.monotonic() + error = api.upload(alice, chat, SAMPLE, 503).json() + assert error["error"]["code"] == "unavailable" + assert time.monotonic() - started < 8 + started = time.monotonic() + api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + assert time.monotonic() - started < 8 + api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only + api.send(alice, chat, "текст работает без S3") # degraded mode +> got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +student-tests\test_storage_failures.py:53: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = , method = 'GET' +path = '/api/v1/attachments/57b6e686-85b7-4dd3-8e02-076b19a8ca79/content' +expected = 200, token = 'S12UUVFiNSnc1NKWb-5PmdJBo9_J6P1-5TihoVd7xBs' +json_body = None, raw = None, headers = None, schema = None, lb_error_ok = False + + def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None, + json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None, + lb_error_ok: bool = False): + """lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is + accepted without X-Instance-Id; every response of an API replica must carry it.""" + h = dict(headers or {}) + if token is not None: + h["Authorization"] = "Bearer " + token + if json_body is not None: + raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8") + h.setdefault("Content-Type", "application/json") + response = self.client.request(method, path, content=raw, headers=h) + allowed = (expected,) if isinstance(expected, int) else expected +> assert response.status_code in allowed, (method, path, response.status_code, response.text[:300]) +E AssertionError: ('GET', '/api/v1/attachments/57b6e686-85b7-4dd3-8e02-076b19a8ca79/content', 503, '{"error":{"code":"unavailable","message":"Хранилище файлов временно недоступно","request_id":"8ecf9351-b1db-4e6d-99da-89f4de083683"}}') +E assert 503 in (200,) +E + where 503 = .status_code + +student-tests\conftest.py:184: AssertionError +=========================== short test summary info =========================== +FAILED student-tests/test_sessions.py::test_restart_does_not_extend_ttl - Ass... +FAILED student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop] +============ 2 failed, 314 passed, 4 skipped in 1229.88s (0:20:29) ============ +[exit 1, 1231.4 s] diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..f9a67d1d --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,331 @@ +# Own tests after fixes found by the first clean-clone run +# 2026-10-25T12:12:45Z, commit fc50362 bench: SQL синтетики через stdin psql (лимит командной строки Windows) +$ python -m pytest student-tests -v +============================= test session starts ============================= +platform win32 -- Python 3.14.6, pytest-9.1.1, pluggy-1.7.0 -- C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe +rootdir: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean +plugins: anyio-4.15.1 +collecting ... collected 320 items + +student-tests/test_api.py::test_register_never_returns_password_and_login_returns_bearer PASSED [ 0%] +student-tests/test_api.py::test_duplicate_username_is_409 PASSED [ 0%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[short-11ch] PASSED [ 0%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[ ] PASSED [ 1%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[ \t \n \t \n \t \n ] PASSED [ 1%] +student-tests/test_api.py::test_bad_password_on_registration_is_400[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 1%] +student-tests/test_api.py::test_edge_passwords_register_and_login[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 2%] +student-tests/test_api.py::test_edge_passwords_register_and_login[with:colons:inside] PASSED [ 2%] +student-tests/test_api.py::test_edge_passwords_register_and_login[Twelve chars] PASSED [ 2%] +student-tests/test_api.py::test_wrong_password_and_unknown_user_give_the_same_401 PASSED [ 3%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic \xe9] PASSED [ 3%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic !!!] PASSED [ 3%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic bm9fY29sb24=] PASSED [ 4%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic //46eA==] PASSED [ 4%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 4%] +student-tests/test_api.py::test_malformed_basic_is_401_not_500[] PASSED [ 5%] +student-tests/test_api.py::test_logout_revokes_only_this_token PASSED [ 5%] +student-tests/test_api.py::test_x_user_id_basic_and_garbage_do_not_authenticate PASSED [ 5%] +student-tests/test_api.py::test_malformed_json_is_400[{] PASSED [ 5%] +student-tests/test_api.py::test_malformed_json_is_400[] PASSED [ 6%] +student-tests/test_api.py::test_malformed_json_is_400[[]] PASSED [ 6%] +student-tests/test_api.py::test_malformed_json_is_400[null] PASSED [ 6%] +student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}] PASSED [ 7%] +student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}] PASSED [ 7%] +student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "\ud800", "password": "Twelve-chars"}] PASSED [ 7%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body0] PASSED [ 8%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body1] PASSED [ 8%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body2] PASSED [ 8%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body3] PASSED [ 9%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body4] PASSED [ 9%] +student-tests/test_api.py::test_invalid_registration_fields_are_400[body5] PASSED [ 9%] +student-tests/test_api.py::test_unicode_limits_are_code_points_and_nul_is_stored PASSED [ 10%] +student-tests/test_api.py::test_oversized_body_and_wrong_content_type PASSED [ 10%] +student-tests/test_api.py::test_chat_validation PASSED [ 10%] +student-tests/test_api.py::test_foreign_chat_is_indistinguishable_from_missing PASSED [ 10%] +student-tests/test_api.py::test_chat_list_order_pagination_and_isolation PASSED [ 11%] +student-tests/test_api.py::test_history_pages_without_loss_or_duplicates PASSED [ 11%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=0] PASSED [ 11%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=101] PASSED [ 12%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=abc] PASSED [ 12%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=1&limit=2] PASSED [ 12%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=] PASSED [ 13%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=garbage] PASSED [ 13%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDF8Mg] PASSED [ 13%] +student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDk5OTk5OTk5OTk5OTk5OTk5OTk5fHk=] PASSED [ 14%] +student-tests/test_api.py::test_cursor_bound_to_chat PASSED [ 14%] +student-tests/test_api.py::test_read_after_write PASSED [ 14%] +student-tests/test_api.py::test_malformed_cursor_values_are_400_on_both_lists PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[\xff] PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"first","TEXT":"second"}] PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"a","text":"b"}] PASSED [ 15%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":123}] PASSED [ 16%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}}] PASSED [ 16%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}]] PASSED [ 16%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x"} {}] PASSED [ 17%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}] PASSED [ 17%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":null}] PASSED [ 17%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":["nope"]}] PASSED [ 18%] +student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":" \n\t "}] PASSED [ 18%] +student-tests/test_api.py::test_ff_byte_body_on_registration_is_contract_error PASSED [ 18%] +student-tests/test_api.py::test_text_is_stored_verbatim_and_length_counts_raw_text PASSED [ 19%] +student-tests/test_api.py::test_unknown_attachment_ids_are_404 PASSED [ 19%] +student-tests/test_api.py::test_member_ids_are_compared_as_uuids_not_strings PASSED [ 19%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer .] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer a b] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 20%] +student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[bearer\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 21%] +student-tests/test_api.py::test_login_with_a_body_is_rejected PASSED [ 21%] +student-tests/test_api.py::test_password_range_is_12_to_128_code_points PASSED [ 21%] +student-tests/test_api.py::test_every_api_response_carries_instance_and_request_id PASSED [ 22%] +student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\xa0] PASSED [ 22%] +student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\u2003\u2003] PASSED [ 22%] +student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[ \xa0\t\u3000] PASSED [ 23%] +student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":"x","password":1e1000}] PASSED [ 23%] +student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"ff_probe","display_name":"x","password":"12345678901\xff"}] PASSED [ 23%] +student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}] PASSED [ 24%] +student-tests/test_api.py::test_more_than_100_members_is_400 PASSED [ 24%] +student-tests/test_api.py::test_cursor_of_one_list_is_rejected_by_the_other PASSED [ 24%] +student-tests/test_attachments.py::test_png_and_jpeg_roundtrip_byte_for_byte PASSED [ 25%] +student-tests/test_attachments.py::test_metadata_reveals_no_storage_details PASSED [ 25%] +student-tests/test_attachments.py::test_original_is_not_stored_on_api_containers PASSED [ 25%] +student-tests/test_attachments.py::test_outsider_and_unknown_get_404 PASSED [ 25%] +student-tests/test_attachments.py::test_attachment_ids_are_checked_per_chat PASSED [ 26%] +student-tests/test_attachments.py::test_message_with_attachments_and_history PASSED [ 26%] +student-tests/test_attachments.py::test_idempotent_replay_with_attachments PASSED [ 26%] +student-tests/test_attachments.py::test_message_body_validation[{"attachment_ids":["%(a)s"]}] PASSED [ 27%] +student-tests/test_attachments.py::test_message_body_validation[{"text":null,"attachment_ids":["%(a)s"]}] PASSED [ 27%] +student-tests/test_attachments.py::test_message_body_validation[{"text":1,"attachment_ids":["%(a)s"]}] PASSED [ 27%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":null}] PASSED [ 28%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":"%(a)s"}] PASSED [ 28%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":{}}] PASSED [ 28%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s","%] PASSED [ 29%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s",1]] PASSED [ 29%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["not-a-uui] PASSED [ 29%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"","attachment_ids":[]}] PASSED [ 30%] +student-tests/test_attachments.py::test_message_body_validation[{"text":" "}] PASSED [ 30%] +student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":[],"sender_] PASSED [ 30%] +student-tests/test_attachments.py::test_at_most_ten_attachments PASSED [ 30%] +student-tests/test_attachments.py::test_size_limit_through_the_load_balancer PASSED [ 31%] +student-tests/test_attachments.py::test_pixel_limit_before_decoding PASSED [ 31%] +student-tests/test_attachments.py::test_exactly_40_megapixels_is_accepted PASSED [ 31%] +student-tests/test_attachments.py::test_bad_content_is_415[png: first 33 bytes of the fixture] PASSED [ 32%] +student-tests/test_attachments.py::test_bad_content_is_415[png: truncated] PASSED [ 32%] +student-tests/test_attachments.py::test_bad_content_is_415[png: corrupt IDAT, valid CRC] PASSED [ 32%] +student-tests/test_attachments.py::test_bad_content_is_415[png: fewer rows than IHDR] PASSED [ 33%] +student-tests/test_attachments.py::test_bad_content_is_415[png: flipped byte] PASSED [ 33%] +student-tests/test_attachments.py::test_bad_content_is_415[png: bytes after IEND] PASSED [ 33%] +student-tests/test_attachments.py::test_bad_content_is_415[jpeg: no scan data after SOS] PASSED [ 34%] +student-tests/test_attachments.py::test_bad_content_is_415[jpeg: truncated] PASSED [ 34%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: png bytes declared jpeg] PASSED [ 34%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: jpeg bytes declared png] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: gif declared png] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[mime: html declared png] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[type: text/plain] PASSED [ 35%] +student-tests/test_attachments.py::test_bad_content_is_415[type: octet-stream] PASSED [ 36%] +student-tests/test_attachments.py::test_bad_content_is_415[type: multipart] PASSED [ 36%] +student-tests/test_attachments.py::test_bad_content_is_415[type: missing] PASSED [ 36%] +student-tests/test_attachments.py::test_empty_body_is_400 PASSED [ 37%] +student-tests/test_attachments.py::test_bad_filename_is_400[missing] PASSED [ 37%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=] PASSED [ 37%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=%20%20%20] PASSED [ 38%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=.] PASSED [ 38%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=..] PASSED [ 38%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%2Fb.png] PASSED [ 39%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%5Cb.png] PASSED [ 39%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%00b.png] PASSED [ 39%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=c1%C2%85.png] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=tab%09.png] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=photo%E2%80%AEgnp.exe] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=%FF%FE.png] PASSED [ 40%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=a.png&filename=b.png] PASSED [ 41%] +student-tests/test_attachments.py::test_bad_filename_is_400[filename=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 41%] +student-tests/test_attachments.py::test_unicode_filename_is_kept_and_safely_exposed PASSED [ 41%] +student-tests/test_attachments.py::test_variant_parameter PASSED [ 42%] +student-tests/test_balancing.py::test_one_session_is_served_by_every_real_replica PASSED [ 42%] +student-tests/test_balancing.py::test_write_on_one_replica_is_read_on_the_other PASSED [ 42%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-pg_hba] PASSED [ 43%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-network] PASSED [ 43%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-pg_hba] PASSED [ 43%] +student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-network] PASSED [ 44%] +student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[0] PASSED [ 44%] +student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[1] PASSED [ 44%] +student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[0] PASSED [ 45%] +student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[1] PASSED [ 45%] +student-tests/test_balancing.py::test_graceful_shutdown_order_in_logs PASSED [ 45%] +student-tests/test_balancing.py::test_full_restart_lb_rediscovers_all_replicas PASSED [ 45%] +student-tests/test_balancing.py::test_scale_up_and_down_is_picked_up_without_errors PASSED [ 46%] +student-tests/test_balancing.py::test_compose_policy_limits_ports_secrets_images PASSED [ 46%] +student-tests/test_balancing.py::test_running_containers_have_limits_and_no_bypass_ports PASSED [ 46%] +student-tests/test_balancing.py::test_migrations_ran_once_by_the_migrate_service PASSED [ 47%] +student-tests/test_consistency.py::test_interrupted_and_stalled_uploads_leave_nothing PASSED [ 47%] +student-tests/test_consistency.py::test_crash_between_s3_put_and_commit_is_cleaned_by_janitor PASSED [ 47%] +student-tests/test_consistency.py::test_lost_answer_after_commit_never_deletes_the_object PASSED [ 48%] +student-tests/test_consistency.py::test_fresh_pending_upload_is_not_touched_by_janitor PASSED [ 48%] +student-tests/test_db.py::test_registration_race_exactly_one_201_and_seven_409 PASSED [ 48%] +student-tests/test_db.py::test_create_chat_rolls_back_after_the_chat_row_was_inserted PASSED [ 49%] +student-tests/test_db.py::test_migrations_rerun_is_a_noop PASSED [ 49%] +student-tests/test_db.py::test_100_messages_from_10_clients_keep_order_and_cursor_walk PASSED [ 49%] +student-tests/test_db.py::test_secrets_are_not_stored_in_plain_text PASSED [ 50%] +student-tests/test_explain.py::test_late_page_uses_index_range_not_filter PASSED [ 50%] +student-tests/test_failures.py::test_valkey_failure[stop] PASSED [ 50%] +student-tests/test_failures.py::test_valkey_failure[pause] PASSED [ 50%] +student-tests/test_failures.py::test_postgres_failure[stop] PASSED [ 51%] +student-tests/test_failures.py::test_postgres_failure[pause] PASSED [ 51%] +student-tests/test_failures.py::test_locked_table_gives_bounded_503 PASSED [ 51%] +student-tests/test_failures.py::test_logout_during_valkey_outage_is_never_lost PASSED [ 52%] +student-tests/test_idempotency.py::test_replay_returns_the_same_201_and_message_from_every_replica PASSED [ 52%] +student-tests/test_idempotency.py::test_other_body_with_the_same_key_is_409_and_keeps_the_original PASSED [ 52%] +student-tests/test_idempotency.py::test_key_order_and_missing_attachment_ids_are_the_same_body PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[] PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[\u043a\u043b\u044e\u0447] PASSED [ 54%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a b] PASSED [ 54%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a/b] PASSED [ 54%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a,b] PASSED [ 55%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a\tb] PASSED [ 55%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[*] PASSED [ 55%] +student-tests/test_idempotency.py::test_repeated_key_header_is_400 PASSED [ 55%] +student-tests/test_idempotency.py::test_longest_key_and_every_allowed_character PASSED [ 56%] +student-tests/test_idempotency.py::test_scope_is_user_plus_chat_plus_key PASSED [ 56%] +student-tests/test_idempotency.py::test_outsider_and_revoked_session_never_see_the_stored_result PASSED [ 56%] +student-tests/test_idempotency.py::test_concurrent_same_key_through_different_replicas_creates_exactly_one PASSED [ 57%] +student-tests/test_idempotency.py::test_concurrent_same_key_different_bodies_one_wins PASSED [ 57%] +student-tests/test_idempotency.py::test_failure_after_the_key_insert_rolls_everything_back PASSED [ 57%] +student-tests/test_idempotency.py::test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message PASSED [ 58%] +student-tests/test_idempotency.py::test_lock_timeout_is_a_transient_503_that_is_not_recorded PASSED [ 58%] +student-tests/test_idempotency.py::test_replay_after_the_serving_replica_is_stopped PASSED [ 58%] +student-tests/test_reconcile.py::test_detect_and_safely_fix_faults_on_both_sides PASSED [ 59%] +student-tests/test_reconcile.py::test_checksum_mode_finds_same_size_corruption PASSED [ 59%] +student-tests/test_reconcile.py::test_safety_valve_refuses_mass_deletion PASSED [ 59%] +student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[s3] PASSED [ 60%] +student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[postgres] PASSED [ 60%] +student-tests/test_reconcile.py::test_live_uploads_are_never_hurt_by_concurrent_apply PASSED [ 60%] +student-tests/test_sessions.py::test_short_ttl_expires[valkey] PASSED [ 60%] +student-tests/test_sessions.py::test_short_ttl_expires[postgres] PASSED [ 61%] +student-tests/test_sessions.py::test_restart_does_not_extend_ttl PASSED [ 61%] +student-tests/test_sessions.py::test_postgres_session_store PASSED [ 61%] +student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop] PASSED [ 62%] +student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[pause] PASSED [ 62%] +student-tests/test_storage_failures.py::test_upload_survives_failover_of_the_serving_replica PASSED [ 62%] +student-tests/test_storage_failures.py::test_bucket_is_private_outside_the_api PASSED [ 63%] +student-tests/test_storage_failures.py::test_s3_restart_keeps_objects PASSED [ 63%] +student-tests/unit/test_images.py::test_public_fixture_is_accepted PASSED [ 63%] +student-tests/unit/test_images.py::test_valid_png_kinds[rgba] PASSED [ 64%] +student-tests/unit/test_images.py::test_valid_png_kinds[palette] PASSED [ 64%] +student-tests/unit/test_images.py::test_valid_png_kinds[1bit] PASSED [ 64%] +student-tests/unit/test_images.py::test_valid_png_kinds[gray] PASSED [ 65%] +student-tests/unit/test_images.py::test_valid_png_kinds[noise] PASSED [ 65%] +student-tests/unit/test_images.py::test_valid_png_kinds[1x1] PASSED [ 65%] +student-tests/unit/test_images.py::test_interlaced_png_is_accepted PASSED [ 65%] +student-tests/unit/test_images.py::test_broken_png_is_415[first 33 bytes of the fixture] PASSED [ 66%] +student-tests/unit/test_images.py::test_broken_png_is_415[truncated (no IEND)] PASSED [ 66%] +student-tests/unit/test_images.py::test_broken_png_is_415[truncated in IDAT] PASSED [ 66%] +student-tests/unit/test_images.py::test_broken_png_is_415[flipped byte -> CRC error] PASSED [ 67%] +student-tests/unit/test_images.py::test_broken_png_is_415[garbage IDAT with valid CRC] PASSED [ 67%] +student-tests/unit/test_images.py::test_broken_png_is_415[fewer rows than IHDR] PASSED [ 67%] +student-tests/unit/test_images.py::test_broken_png_is_415[bytes after IEND (polyglot)] PASSED [ 68%] +student-tests/unit/test_images.py::test_broken_png_is_415[text] PASSED [ 68%] +student-tests/unit/test_images.py::test_broken_png_is_415[gif] PASSED [ 68%] +student-tests/unit/test_images.py::test_broken_png_is_415[two bytes of a signature] PASSED [ 69%] +student-tests/unit/test_images.py::test_pixel_limit_is_checked_from_the_header PASSED [ 69%] +student-tests/unit/test_images.py::test_png_verification_is_streaming_and_fast PASSED [ 69%] +student-tests/unit/test_images.py::test_jpeg_header_is_parsed_incrementally PASSED [ 70%] +student-tests/unit/test_images.py::test_valid_jpeg[base] SKIPPED (dj...) [ 70%] +student-tests/unit/test_images.py::test_valid_jpeg[prog] SKIPPED (dj...) [ 70%] +student-tests/unit/test_images.py::test_broken_jpeg_is_415[no scan data after SOS-] SKIPPED [ 70%] +student-tests/unit/test_images.py::test_broken_jpeg_is_415[truncated-] SKIPPED [ 71%] +student-tests/unit/test_images.py::test_good_filenames[photo.png] PASSED [ 71%] +student-tests/unit/test_images.py::test_good_filenames[\u0444\u043e\u0442\u043e \u043e\u0442\u043f\u0443\u0441\u043a\u0430.jpg] PASSED [ 71%] +student-tests/unit/test_images.py::test_good_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 72%] +student-tests/unit/test_images.py::test_good_filenames[emoji \U0001f468\u200d\U0001f469\u200d\U0001f467.png] PASSED [ 72%] +student-tests/unit/test_images.py::test_good_filenames[ x ] PASSED [ 72%] +student-tests/unit/test_images.py::test_good_filenames[..x] PASSED [ 73%] +student-tests/unit/test_images.py::test_bad_filenames[] PASSED [ 73%] +student-tests/unit/test_images.py::test_bad_filenames[ ] PASSED [ 73%] +student-tests/unit/test_images.py::test_bad_filenames[\u3000] PASSED [ 74%] +student-tests/unit/test_images.py::test_bad_filenames[.] PASSED [ 74%] +student-tests/unit/test_images.py::test_bad_filenames[..] PASSED [ 74%] +student-tests/unit/test_images.py::test_bad_filenames[a/b.png] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[a\\b.png] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[a\x00b] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[c1\x85.png] PASSED [ 75%] +student-tests/unit/test_images.py::test_bad_filenames[tab\t.png] PASSED [ 76%] +student-tests/unit/test_images.py::test_bad_filenames[del\x7f.png] PASSED [ 76%] +student-tests/unit/test_images.py::test_bad_filenames[x\u2028y] PASSED [ 76%] +student-tests/unit/test_images.py::test_bad_filenames[photo\u202egnp.exe] PASSED [ 77%] +student-tests/unit/test_images.py::test_bad_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 77%] +student-tests/unit/test_images.py::test_filename_must_be_single_and_present PASSED [ 77%] +student-tests/unit/test_images.py::test_query_must_be_valid_utf8 PASSED [ 78%] +student-tests/unit/test_unit.py::test_valid_passwords[Twelve-chars] PASSED [ 78%] +student-tests/unit/test_unit.py::test_valid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 78%] +student-tests/unit/test_unit.py::test_valid_passwords[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u044f\u043c\u0438] PASSED [ 79%] +student-tests/unit/test_unit.py::test_valid_passwords[\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600] PASSED [ 79%] +student-tests/unit/test_unit.py::test_valid_passwords[a b c d e f g] PASSED [ 79%] +student-tests/unit/test_unit.py::test_invalid_passwords[short-11ch] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[ ] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000] PASSED [ 80%] +student-tests/unit/test_unit.py::test_invalid_passwords[\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800] PASSED [ 81%] +student-tests/unit/test_unit.py::test_invalid_passwords[None] PASSED [ 81%] +student-tests/unit/test_unit.py::test_invalid_passwords[123456789012] PASSED [ 81%] +student-tests/unit/test_unit.py::test_text_may_contain_nul_but_not_lone_surrogates PASSED [ 82%] +student-tests/unit/test_unit.py::test_cursor_roundtrip_and_binding PASSED [ 82%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|253402300800000000|11111111-1111-4111-8111-111111111111] PASSED [ 82%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|9999999999999999999|11111111-1111-4111-8111-111111111111] PASSED [ 83%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|-1|11111111-1111-4111-8111-111111111111] PASSED [ 83%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|not-a-uuid] PASSED [ 83%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-11111111111G] PASSED [ 84%] +student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v2|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-111111111111] PASSED [ 84%] +student-tests/unit/test_unit.py::test_time_helpers_are_exact PASSED [ 84%] +student-tests/unit/test_unit.py::test_basic_is_parsed_and_password_may_contain_colons PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[None] PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[] PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic] PASSED [ 85%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic \xe9] PASSED [ 86%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic !!!!] PASSED [ 86%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic bm8tY29sb24=] PASSED [ 86%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic //46cHc=] PASSED [ 87%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic OnBhc3N3b3Jk] PASSED [ 87%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic YWxpY2U6] PASSED [ 87%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic 0JDQu9C40YHQsDpwdw==] PASSED [ 88%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 88%] +student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA] PASSED [ 88%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[None] PASSED [ 89%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[] PASSED [ 89%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer] PASSED [ 89%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer short] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Basic YWxpY2U6cGFzc3dvcmQtMTIz] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 90%] +student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 91%] +student-tests/unit/test_unit.py::test_bearer_ok PASSED [ 91%] +student-tests/unit/test_unit.py::test_argon2id_hash_and_verify PASSED [ 91%] +student-tests/unit/test_unit.py::test_s3_settings_are_validated_and_contract_limits_cannot_be_raised PASSED [ 92%] +student-tests/unit/test_unit.py::test_settings_read_secret_files_and_contract_names PASSED [ 92%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[a] PASSED [ 92%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 93%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[A-Za-z0-9._:-] PASSED [ 93%] +student-tests/unit/test_unit.py::test_valid_idempotency_keys[2026-10-16T10:00:00.1] PASSED [ 93%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[None] PASSED [ 94%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[] PASSED [ 94%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 94%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\u043a\u043b\u044e\u0447] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a b] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a/b] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a,b] PASSED [ 95%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\tb] PASSED [ 96%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\x00b] PASSED [ 96%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[*] PASSED [ 96%] +student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\xe9] PASSED [ 97%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"a","text":"b"}] PASSED [ 97%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"a":{"x":1,"x":2}}] PASSED [ 97%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[\xff] PASSED [ 98%] +student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"x"} {}] PASSED [ 98%] +student-tests/unit/test_unit.py::test_attachment_ids_absent_equals_empty_and_is_validated PASSED [ 98%] +student-tests/unit/test_unit.py::test_request_fingerprint_ignores_key_order_and_default_attachments PASSED [ 99%] +student-tests/unit/test_unit.py::test_instance_id_defaults_to_hostname_and_is_validated PASSED [ 99%] +student-tests/unit/test_unit.py::test_idempotency_ttl_cannot_be_below_24_hours PASSED [ 99%] +student-tests/unit/test_unit.py::test_message_write_path_setting PASSED [100%] + +================= 316 passed, 4 skipped in 1269.76s (0:21:09) ================= diff --git a/evidence/test-failover.txt b/evidence/test-failover.txt new file mode 100644 index 00000000..c17eccf2 --- /dev/null +++ b/evidence/test-failover.txt @@ -0,0 +1,74 @@ +# make test-failover with scripts/stop-one: which container is really stopped, restore, repeat +# 2026-10-25T11:42:11Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов + +--- run 1; running API replicas BEFORE: +mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=33790 started=2026-10-25T11:41:59.187 +mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=33848 started=2026-10-25T11:41:59.568 + +$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd +python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 5.956} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +[exit 0, 16.5 s] + +AFTER (the stopped replica, left stopped by the hook): +mtusi-lab4-api-1 host=8691c30aa702 exited exit=0 pid=0 started=2026-10-25T11:41:59.187 + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status +api2 172.25.0.3 UP mtusi-lab4-api-2 X-Instance-Id=4026e296947c +(other slots: MAINT, no container) +[exit 0, 0.7 s] + +$ sh scripts/restore-api.sh +mtusi-lab4-api-1 Up 6 seconds (healthy) +mtusi-lab4-api-2 Up 37 seconds (healthy) + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-1 Starting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy +[exit 0, 6.9 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2 +2 API replica(s) UP in HAProxy after 0.6 s: ['4026e296947c', '8691c30aa702'] +[exit 0, 1.4 s] + +--- run 2; running API replicas BEFORE: +mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=35544 started=2026-10-25T11:42:30.519 +mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=33848 started=2026-10-25T11:41:59.568 + +$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd +python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 6.291} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +[exit 0, 16.4 s] + +AFTER (the stopped replica, left stopped by the hook): +mtusi-lab4-api-2 host=4026e296947c exited exit=0 pid=0 started=2026-10-25T11:41:59.568 + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status +api1 172.25.0.2 UP mtusi-lab4-api-1 X-Instance-Id=8691c30aa702 +(other slots: MAINT, no container) +[exit 0, 0.8 s] + +$ sh scripts/restore-api.sh +mtusi-lab4-api-1 Up 32 seconds (healthy) +mtusi-lab4-api-2 Up 6 seconds (healthy) + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Starting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy +[exit 0, 6.9 s] + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2 +2 API replica(s) UP in HAProxy after 0.6 s: ['4026e296947c', '8691c30aa702'] +[exit 0, 1.5 s] + +Replicas stopped by the public test across runs: ['mtusi-lab4-api-1', 'mtusi-lab4-api-2'] diff --git a/evidence/test-persistence.txt b/evidence/test-persistence.txt new file mode 100644 index 00000000..bc584f1e --- /dev/null +++ b/evidence/test-persistence.txt @@ -0,0 +1,32 @@ +# make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3 +# 2026-10-25T11:41:08Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов + +Containers BEFORE: +mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=28760 started=2026-10-25T11:40:06.635 +mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=28857 started=2026-10-25T11:40:07.085 +mtusi-lab4-lb-1 host=1282a32916ca running exit=0 pid=29105 started=2026-10-25T11:40:13.103 +mtusi-lab4-migrate-1 host=62f12280b06e exited exit=0 pid=0 started=2026-10-25T11:39:59.135 +mtusi-lab4-postgres-1 host=286ea382ba46 running exit=0 pid=28093 started=2026-10-25T11:39:53.269 +mtusi-lab4-s3-1 host=53564c9a38b4 running exit=0 pid=28181 started=2026-10-25T11:39:53.656 +mtusi-lab4-s3-init-1 host=ab8ae8b24a67 exited exit=0 pid=0 started=2026-10-25T11:40:01.363 +mtusi-lab4-valkey-1 host=44817703634f running exit=0 pid=28258 started=2026-10-25T11:39:53.966 + +$ make test-persistence ACTION_SCRIPT=scripts/restart.cmd +python tests/resilience.py --mode persistence --action-script "scripts/restart.cmd" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +[exit 0, 61.5 s] + +Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept): +mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=33790 started=2026-10-25T11:41:59.187 +mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=33848 started=2026-10-25T11:41:59.568 +mtusi-lab4-lb-1 host=1282a32916ca running exit=0 pid=34447 started=2026-10-25T11:42:09.092 +mtusi-lab4-migrate-1 host=62f12280b06e exited exit=0 pid=0 started=2026-10-25T11:42:01.016 +mtusi-lab4-postgres-1 host=286ea382ba46 running exit=0 pid=32342 started=2026-10-25T11:41:15.594 +mtusi-lab4-s3-1 host=53564c9a38b4 running exit=0 pid=32960 started=2026-10-25T11:41:35.722 +mtusi-lab4-s3-init-1 host=ab8ae8b24a67 exited exit=0 pid=0 started=2026-10-25T11:42:03.276 +mtusi-lab4-valkey-1 host=44817703634f running exit=0 pid=32296 started=2026-10-25T11:41:15.515 + +$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2 +2 API replica(s) UP in HAProxy after 0.5 s: ['4026e296947c', '8691c30aa702'] +[exit 0, 1.2 s] diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..13172af4 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,116 @@ +# HAProxy 3.2 LTS — the single public entry point of the lab 4 stand (compose service `lb`). +# +# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the +# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and +# re-resolves every second, so scale up/down, a restarted container with a new IP and a +# full stack restart are picked up without restarting the LB. A stopped container drops +# out of DNS and its slot goes to MAINT. +# +# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey + S3 reachable, not +# draining) every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide +# anything on their own, and there are NO fallback servers without checks: a replica that +# answers /health/live but not /health/ready never gets traffic while a fully ready one exists. +# +# Degraded mode (lab 4): S3 is a SHARED dependency. If it is down, /health/ready is 503 on +# every replica and backend `api` has no server. Instead of answering 503 to everything, the +# frontend then switches to backend `api_core`: the same replicas, checked by +# /health/ready/core (PostgreSQL + Valkey, not draining). Chats and text messages keep working, +# every attachment request is answered 503 by the API itself (fast, S3_TIMEOUT_SECONDS). +# A replica that alone lost S3 (e.g. its storage network) is DOWN in `api` while the others are +# UP, so it gets no traffic at all. +# Worst case detection of a failed replica (a check answered at once counts at once, a check +# that hangs counts only after `timeout check`): +# fast explicit 503 (drain after SIGTERM): inter 1 s + fastinter 0.5 s ~= 1.5 s; +# slow ready (PostgreSQL/Valkey unreachable: the answer takes ~1 s = timeout check) +# or frozen process: inter 1 s + 1 s + fastinter 0.5 s + 1 s ~= 3.5 s. +# Requests that could not even be SENT (connection refused/reset before the request) +# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST, +# because the API has not seen them. Nothing that may have reached the API is retried by +# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key. +# No sticky sessions: sessions live in Valkey, every replica serves every user. + +global + log stdout format raw local0 info + maxconn 4096 + # Runtime API for `show servers state` / `show stat` (scripts and tests, no network port). + stats socket /var/lib/haproxy/admin.sock mode 600 level admin + hard-stop-after 15s + +resolvers docker + nameserver dns1 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + +defaults + mode http + log global + option httplog + option dontlognull + option http-keep-alive + timeout connect 1s + timeout client 30s + timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 3s + timeout check 1s + retries 2 + retry-on conn-failure + option redispatch 1 + # Errors generated by the LB itself, in the contract error format with the same request id. + # They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the + # request is unknown (a replica died mid-request / did not answer in time): retry with the + # same Idempotency-Key. 503 = no ready replica at all. + http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + +frontend public + bind :8080 + # One request id end to end: keep a valid client X-Request-Id, otherwise create one; the + # API echoes it, and errors generated by the LB itself carry the same value. + http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ } + http-request set-var(txn.request_id) req.hdr(x-request-id) + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\"" + use_backend api if { nbsrv(api) gt 0 } + default_backend api_core + +backend api + balance roundrobin + # Uploads (<= 10 MiB body + verification + S3 PUT): the API's own budget is + # UPLOAD_READ_TIMEOUT 30 s + IMAGE_VERIFY 10 s + S3; the LB must not cut it at 10 s (a 504 + # after a successful upload would make the client upload a duplicate). + acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 60s if is_upload METH_POST + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200 + server-template api 8 api:8080 + +# Degraded mode only (see the header): used when NO replica is fully ready. +backend api_core + balance roundrobin + acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 60s if is_upload METH_POST + option httpchk + http-check send meth GET uri /health/ready/core ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200 + server-template core 8 api:8080 + +# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now. +frontend stats + bind :8404 + stats enable + stats uri / + stats refresh 2s + stats show-legends diff --git a/infra/seaweedfs/start.sh b/infra/seaweedfs/start.sh new file mode 100755 index 00000000..ad4f09c5 --- /dev/null +++ b/infra/seaweedfs/start.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume + +# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`. +# +# Identities (SeaweedFS s3.json, actions scoped to the bucket): +# bootstrap-admin Admin (create bucket) -> only s3-init has these keys +# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin) +# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing) +# There is NO anonymous identity, so every unsigned request is refused: the bucket is private. +# +# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on +# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the +# container network `storage` (internal, not published on the host). Telemetry is off. +set -eu +umask 077 +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +bucket="${S3_BUCKET:?S3_BUCKET is required}" +cat > /tmp/s3.json < ($2, $3) ORDER BY created_at, id LIMIT n +-- becomes an index range scan that starts exactly at the cursor. +CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id); + +-- Sessions for SESSION_STORE=postgres (the default store is Valkey; see docs/DATA_MODEL.md). +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, -- SHA-256 of the token; the token is not stored + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32) +); +CREATE INDEX sessions_expires_at ON sessions (expires_at); diff --git a/migrations/0002_idempotency_keys.sql b/migrations/0002_idempotency_keys.sql new file mode 100644 index 00000000..108dafde --- /dev/null +++ b/migrations/0002_idempotency_keys.sql @@ -0,0 +1,25 @@ +-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages. +-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe +-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции"). +-- +-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key. +-- The method (POST) and the path template are fixed for this table, so the scope +-- is (user_id, chat_id, key). The key row, the message and the result are written in +-- ONE transaction (repo.post_message): there is no "pending" state that a crash could +-- leave behind, and the stored result is the message row itself. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id), + key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body + -- Inserted before the message in the same transaction, hence checked at COMMIT. + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_keys_message_key UNIQUE (message_id) +); +-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h). +CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at); diff --git a/migrations/0003_post_message_function.sql b/migrations/0003_post_message_function.sql new file mode 100644 index 00000000..268a0c0a --- /dev/null +++ b/migrations/0003_post_message_function.sql @@ -0,0 +1,50 @@ +-- 0003 (lab 3, track 1 improvement): the whole idempotent "send message" as ONE statement. +-- Additive (a new function), so replicas of the previous version keep working while it is +-- applied, and the previous write path (MESSAGE_WRITE_PATH=multi) stays usable for A/B runs. +-- +-- Same semantics and the same SQL as repo.post_message (multi-statement path), but executed +-- inside the server: the API sends one call instead of ~11 round trips in two transactions +-- (membership check, BEGIN, key claim, chat clock, message, COMMIT, pool checks). Called in +-- autocommit mode, so the function call IS the transaction: key row, chat clock and message +-- commit together or not at all. VOLATILE + READ COMMITTED: every statement inside takes a +-- fresh snapshot, so after waiting on a concurrent claim of the same key the stored result +-- of the winner is visible, exactly as in the multi-statement path. +-- +-- o_status: created | replayed | conflict | not_found | retry (key vanished in between: cleanup) +CREATE FUNCTION post_message(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid, + p_text bytea, p_ttl_seconds integer) +RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz) +LANGUAGE plpgsql VOLATILE AS $$ +DECLARE + v_created timestamptz; +BEGIN + PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user; + IF NOT FOUND THEN + RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz; + RETURN; + END IF; + + INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, p_message) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() + WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds); + IF FOUND THEN + -- Per-chat clock under the chat row lock (see migrations/0001 and docs/DATA_MODEL.md). + UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') + WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (p_message, p_chat, p_user, p_text, v_created); + RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created; + RETURN; + END IF; + + RETURN QUERY + SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; + IF NOT FOUND THEN + RETURN QUERY SELECT 'retry'::text, NULL::uuid, NULL::bytea, NULL::timestamptz; + END IF; +END +$$; diff --git a/migrations/0004_attachments.sql b/migrations/0004_attachments.sql new file mode 100644 index 00000000..efc1d5fe --- /dev/null +++ b/migrations/0004_attachments.sql @@ -0,0 +1,145 @@ +-- 0004 (lab 4): attachments. Metadata live here, the bytes in the private S3 bucket. +-- Additive (new tables, a relaxed CHECK, a NEW function post_message_v2), so replicas of the +-- lab 3 version keep working while it is applied (they never send attachment_ids). +-- +-- Consistency between the two stores (docs/CONSISTENCY.md): +-- 1. INSERT the row as status='pending' (intent: the server-generated object_key is known +-- BEFORE anything is written to S3) and COMMIT; +-- 2. PUT the object into S3; +-- 3. UPDATE status='ready' WHERE status='pending'; only then 201. +-- Invariant: an object never exists without a row that names its key (row first, object +-- second; on cleanup object first, row second). A crash/outage between the steps leaves a +-- 'pending' row; the janitor deletes its object (idempotent) and then the row once it is older +-- than PENDING_GRACE_SECONDS. The reconciler (python -m messenger.reconcile) checks the +-- invariant over the whole bucket. + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename text NOT NULL, -- display name only, never a path or a key + content_type text NOT NULL, -- detected from the bytes, not from the header + size_bytes integer NOT NULL, + sha256 bytea NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + -- Random server-generated key ("o/" + 128 random bits in hex), unrelated to id/filename. + -- COLLATE "C": byte order = S3 ListObjectsV2 order, the reconciler merge-joins both lists. + object_key text COLLATE "C" NOT NULL, + status text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + -- Set by the reconciler when the object of a READY row is missing or has the wrong size: + -- the row is quarantined (content -> 404, cannot be attached), never deleted, and the mark + -- is removed again if the object comes back (restore from backup). + lost_at timestamptz, + lost_reason text, + -- The uploader must be a member of the chat (enforced by the database too). + CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^o/[0-9a-f]{32}$'), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32), + CONSTRAINT attachments_pixels CHECK (width >= 1 AND height >= 1 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_lost CHECK ((lost_at IS NULL) = (lost_reason IS NULL) AND (lost_at IS NULL OR status = 'ready')) +); +-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads). +CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending'; + +-- attachment_ids of a message, twice on purpose (both written in the message's transaction): +-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the +-- keyset range scan of messages stays ONE index scan, without a per-row sub-query; +-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the +-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id. +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + position smallint NOT NULL, + attachment_id uuid NOT NULL REFERENCES attachments (id), + PRIMARY KEY (message_id, position), + CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9) +); +-- "Is this attachment used by any message?" (janitor/reconciler never touch used ones). +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id); + +-- Lab 4: text may be "" when the message has attachments (the API enforces "text or files"). +-- DROP CONSTRAINT takes a short exclusive lock on messages and the new CHECK scans the table +-- once, inside this migration's transaction (seconds for a lab-sized table; for a big table +-- this would be ADD ... NOT VALID + VALIDATE in a separate migration). +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (octet_length(text) <= 16000); + +-- post_message_v2: post_message (0003) + attachment_ids, still ONE statement and ONE transaction. +-- The attachments are checked BEFORE the key is claimed and the key is claimed only if they are +-- fine, so a failed check never leaves a key behind. A replay (same key, same body) returns the +-- stored message even if an attachment was quarantined since: the result of a key is final. +-- +-- o_status: created | replayed | conflict | not_found | retry | attachment_not_found | attachment_conflict +CREATE FUNCTION post_message_v2(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid, + p_text bytea, p_ttl_seconds integer, p_attachments uuid[]) +RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[]) +LANGUAGE plpgsql VOLATILE AS $$ +DECLARE + v_created timestamptz; + v_found integer; + v_usable integer; + v_check text := 'ok'; +BEGIN + PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user; + IF NOT FOUND THEN + RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; + RETURN; + END IF; + + IF cardinality(p_attachments) > 0 THEN + -- Visible = ready attachment of THIS chat (pending, foreign or unknown -> 404). + -- Usable = visible and not quarantined (lost -> 409, like a failed attachment). + -- FOR SHARE: a concurrent quarantine waits until this message is committed. + SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) INTO v_found, v_usable + FROM (SELECT a.lost_at FROM attachments a + WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status = 'ready' + FOR SHARE) a; + IF v_found <> cardinality(p_attachments) THEN + v_check := 'attachment_not_found'; + ELSIF v_usable <> v_found THEN + v_check := 'attachment_conflict'; + END IF; + END IF; + + IF v_check = 'ok' THEN + INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, p_message) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() + WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds); + IF FOUND THEN + UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') + WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments); + INSERT INTO message_attachments (message_id, position, attachment_id) + SELECT p_message, (a.ord - 1)::smallint, a.id + FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord); + RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments; + RETURN; + END IF; + END IF; + + RETURN QUERY + SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at, + m.attachment_ids + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; + IF NOT FOUND THEN + RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END, + NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; + END IF; +END +$$; diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..fea533d5 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,6 @@ +# Student tests (in-process, via Starlette TestClient). Runtime deps come from requirements.txt. +pytest==9.1.1 +httpx2==2.13.1 +jsonschema==4.26.0 +# Lab 4 tests: generate PNG/JPEG fixtures (valid, corrupt, oversized) on the fly. +pillow==12.3.0 diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..22ea682d --- /dev/null +++ b/requirements.in @@ -0,0 +1,20 @@ +# Direct runtime dependencies. The pinned lock with hashes is requirements.txt: +# docker run --rm -v "$PWD:/w" -w /w python:3.13.16-slim-bookworm \ +# sh -c "pip install -q pip-tools==7.6.1 && pip-compile --generate-hashes --strip-extras --no-emit-index-url -o requirements.txt requirements.in" +starlette==1.7.0 +uvicorn==0.54.0 +# Fast C parser/event loop used by uvicorn in the container (Linux only). +httptools==0.9.0 +uvloop==0.23.0 ; sys_platform != "win32" +# PostgreSQL driver (libpq bundled in the binary wheel) and its async connection pool. +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +# Valkey client (Valkey speaks the Redis protocol). +redis==8.1.0 +# Argon2id password KDF. +argon2-cffi==25.1.0 +# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel). +aiobotocore==3.9.2 +# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo +# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile). diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c7c583c8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,1356 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in +# +aiobotocore==3.9.2 \ + --hash=sha256:363b4892423b272eb84afbb58000da0be6ccd0cf8a68b98438c957d05b841efc \ + --hash=sha256:8e32238c5bd77717ab1ac90077ff51ab141d025528115ccc7cc9b50b2d176e5f + # via -r requirements.in +aiohappyeyeballs==2.7.1 \ + --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + # via aiohttp +aiohttp==3.14.5 \ + --hash=sha256:0133c3c3b54a0bf1e71fa5c1ad95c93f07fd54e24ef1fe182f5122e1573d2bf1 \ + --hash=sha256:038c2c7e8caa26b6c8423779b5eaf1893904048a512c19b32fe841ffa5592b50 \ + --hash=sha256:0684952aeae1f5dbfe02d46039338513b94009baecd15d8e4098a357c4c4a2a6 \ + --hash=sha256:0790ec66fa4013e83c53b9025a45d454723da1a2fce28b3208c9b32d08af162f \ + --hash=sha256:083673c7a94c3ea035caaa5ca04288bdb44887abfe1f5ba23294e6a4b03efd2d \ + --hash=sha256:09e0eb18c7e0c8777e2f9149de63799195b9b3ca1b5c81ba6f32f2c6b8628210 \ + --hash=sha256:09ec102b4b8c9a920275733bbc11fdbb615efe6f9231a06007c0218d336fb77a \ + --hash=sha256:0a8ea271867e360ac985ae607f4a23ad9a38414b9aca1d49ec98839ae660e49f \ + --hash=sha256:0e6f16f5e49c4b8267988c05ab07760d7064cea57d077c3d068d04b0fbb992cb \ + --hash=sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e \ + --hash=sha256:1220353657ad49493551f089ce02f1a348fd57ffd585bfec77f2f3c4fe3a7346 \ + --hash=sha256:137351bf20bbed9a65e839f4a4452ac377389bdb2f2857d2acffef38f5e9f2d1 \ + --hash=sha256:149fb56caf7acb67073126f675d0958d9c4b3125fcd3f6d4877df98aa8a97ce9 \ + --hash=sha256:14f04769cfefe4734016a856a83af36133cd17779cef9ae817f812b8ba9d6d51 \ + --hash=sha256:15a310d3c71398e3d7bfc93a1a73fbe664315cd9e9016b8efc1cff85eeab7155 \ + --hash=sha256:1612fa5857b37bf32e5c1eaeefb96e3b01e9c70679eec81f0934e8a600080863 \ + --hash=sha256:16c8abd5bca220a47efe667d26f8460124c81810787e79ee87b242677563d9dd \ + --hash=sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30 \ + --hash=sha256:1aead151c3abbac6b32942e452020cb66d7efc099d253cc6c20f748e926c858b \ + --hash=sha256:1b438b73c38111818d0c9d6a5c2bfed8584c8e503a49ef085d70e874ec846738 \ + --hash=sha256:1b5416552740edf07234cc9437d0706f2acb67b93c198670b1a68e1b2b587dec \ + --hash=sha256:1d2d981b53dd09a319e3570ef8cc3bbc3ef86f5a7abef0f6b2bff3867db3a9e7 \ + --hash=sha256:1eb8167961ec4dfcc8cb9dd50bd0ee72519f7ef496be95203e49e27b01618382 \ + --hash=sha256:1ffa3a523a36d8628f98c06492ae16a31a23d14c0b4ec721757b477319f656d6 \ + --hash=sha256:20064a177a070d789ee64a50b01a9161d3468e989baacfc6c714aa685c4b332f \ + --hash=sha256:20726f9782d5c2744c1c66255842d1d163bb3edcf768b8de25216bf47f7b6ccf \ + --hash=sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584 \ + --hash=sha256:225c579c23b68b343cccea27a7e06e3bd8ec23a09c30b427eb3f1e4ca6239b20 \ + --hash=sha256:24409db442e2fb6e766bc7f3943851a8381dec3098140e43bb2e843b79e31b12 \ + --hash=sha256:248d779ad720b49d4fb355720e60c9e5f444f95887bc16974fea48fc56c41789 \ + --hash=sha256:2528cb4c6b92008c76ac9ac6298624069bb2db91ff4929905512d1d84485f658 \ + --hash=sha256:276a4fc00b1d9ae492b802763a789c5b86328b989c5ea169f2faa447d6a11c7c \ + --hash=sha256:27c2322e03f66101acb09869ce1cf1efc04994ee95e1735b69827bf8c8b9d781 \ + --hash=sha256:293d3ae7c6a0ed176a42e59a1b5fde825ead65c835360f734148e96729f928d2 \ + --hash=sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111 \ + --hash=sha256:3093b72c215bda16ce961a6d073f6e71d46e022962a9d5d457c5d4d421c78b57 \ + --hash=sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4 \ + --hash=sha256:330900acd0dc4cb8b27f9c127fbaad770964845338493e7906ae3822e82dbf8d \ + --hash=sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab \ + --hash=sha256:3ae800a20947e2c2e53088047d021e6bf7d51560cc49f6a0737a1f79d2e3a13c \ + --hash=sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748 \ + --hash=sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d \ + --hash=sha256:3e51a27980c3788e6e6b3325d694fdd4898087fa8a86b2763af77b39353da41e \ + --hash=sha256:3edbece0379b8b4aaa67619b8aa2399bb66fce372cd5911098a434ea77220aa0 \ + --hash=sha256:3f2dcc00191fd563e9075181a14ec31d7dd63223ced7582cc70a15a499de0c79 \ + --hash=sha256:42b5e616946dbaf505e2bff18c9af2cd4ef9e7ef300ee58a6e951a5b7cf147ae \ + --hash=sha256:42f320d4a5b00b9af0bddcfec5407dc6f2d9816f006b2f79ebbaa31f16895df3 \ + --hash=sha256:43351bdb5e4c3cb7d1772368e988534e869a74db7778079a83782c11c69535c7 \ + --hash=sha256:43e1b7994a8b038125f722bff07492ef501110722c2727c408995d9fb864c421 \ + --hash=sha256:4887d130a7bbfed3a85493bb5a25e5b5b558d40c1d986dd16970d2bb26d63793 \ + --hash=sha256:4f07fe3ac408d8b3f768be471dc3f56d43843c47d97c66120534467a15ead197 \ + --hash=sha256:50343c1757b4b6f6708eeaf24534b32f19dfb99fb1b762c00420867a62fc81e0 \ + --hash=sha256:50983e3be33d8c0942ab88cec3905b10602f64c469b20153c48c5d4e558dd016 \ + --hash=sha256:50a195903119008fe9cc68710535eb37f556ffffd6a7759afe70a2c145587045 \ + --hash=sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178 \ + --hash=sha256:56572c42e3ecd636de8d2c3dd54cf5fc939cb5c32eb56297f176a0d366fac622 \ + --hash=sha256:56d9828f204331a5ca8850fcfe2bcce95a149f1f223f60cc7216e5524978e480 \ + --hash=sha256:579f97d5120f2971876d2ddca2968135f6944d00c44c3a6590ad7d86ca9b403f \ + --hash=sha256:5af42135fdfebdadbc2bcd9c0842a48ccf0d62794c36a260b21dc4b94d1e0119 \ + --hash=sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3 \ + --hash=sha256:5c76f1802bab718a68ac3cce447160605c734551f95c67ae90fa1132b215cb29 \ + --hash=sha256:5e8f97c0488ffda3082766ac0f2c8150a9a58c4d05788330e479cfd449b37939 \ + --hash=sha256:5f3e96071686755d9cd3600c3880183eb94b012178e92746d68101800f0ed8a3 \ + --hash=sha256:5fb6a6e919bfb703227bc1ce6579281b84b1a2ba57deb9794dfdbec7dcd1e40c \ + --hash=sha256:602c1e9b718a3275c580149f947e7fac65044c0a20e599553fb12e9700da9eca \ + --hash=sha256:657291433bf4dd3142f3abac495764cd47d0c7c92087751e6666c6447e65fcef \ + --hash=sha256:6774814fd5c338e72ee0da5cbb9432816df450e69c019f72b5d29bdec2a1792d \ + --hash=sha256:6da32b5ff3fd78d244e37300463434c7145162bfd2b6e9e915ab164da37f7343 \ + --hash=sha256:6e1d8637cf73eebc92eba2e11d4cfff98a3b562f2505bd75bba766d908926e8d \ + --hash=sha256:6e4251c0ba4624a68a2c11471a1ac54c3306876c21f0ae86de085cc9241c8905 \ + --hash=sha256:6f275c11d1aa6d4c458e05a68be084efe3c55a113d99e3f46a318098e52948fc \ + --hash=sha256:6f967dde489ca6a8c02d093ab245d2cbf50ccb5c36adf0188b17b0ca39d24b67 \ + --hash=sha256:6fdcd6af7e2e51d1ba1b4bea16e97b074bcb7b5dd0246a9d8201341bb28085a0 \ + --hash=sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794 \ + --hash=sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835 \ + --hash=sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d \ + --hash=sha256:74efb69332b85675b1eabd760a8cfc2e2cf42c60607c66f88014c1bdfb40942d \ + --hash=sha256:755933b107ea7a6a9ac916f635a70595a5b1a32fac10a8ff0b9f2ab88555550c \ + --hash=sha256:7779cd97e61ebe583ec2f1c5616cdd038aa08a4453b1848c67842176d054948e \ + --hash=sha256:788ecaa9c10533b786ce5ba70c4f2df78ad41819fd00a6c99d92b66f9a32e1da \ + --hash=sha256:7bd8ac754ebd6733a3e2a0dd1674c4d8ab086196803fd8dcd776f07b4e2607d9 \ + --hash=sha256:81c2b3dfd56c62bee6108e4852d5970b4cf9086390b6983f52b666e878c1f115 \ + --hash=sha256:823c910f046f23f4c713b8d99a2242dc65f591cb45ee86418fa11762a3c2963c \ + --hash=sha256:82c7583cd3dfdc7dcc927835b4f6c7faae7ecc1ba3ca5879321621ae2e6f8e84 \ + --hash=sha256:8966ecac808dd5f473c9c4cefd10cd3ffda71c18a4d3493b7c7d2ae1803bf2cc \ + --hash=sha256:8df7d481654ac96fe1ba9a02a9f67770fdd367823e0d5ef01b922725c4bd2cfa \ + --hash=sha256:8e317e0fb6b16212c881d2205a7d87414c29acd69320b3aa6dce9d9c7b86fe4f \ + --hash=sha256:932ce7e694bbc29b2bf6f64f2343c27d148d4997c771d01bdade4639b6749ff4 \ + --hash=sha256:939042d5cda21d41a6f512e7cc8b8e33a2aebff863352251da495fbd91b673b5 \ + --hash=sha256:94684b879ac1d71e4238850c99b62dc1b28d9086b156a2555f082010b85a865c \ + --hash=sha256:96a2e584f0b9ed8f1fa33211397dcf67bb7069866402cb405d191c2f0defb9a3 \ + --hash=sha256:9ad7e6aa38c20da1be697874349c4c273c8a03b7887169665081706398d0439a \ + --hash=sha256:9b42db919715e91eb76acf3bc492a9a7ccd8bd9adc6745c1412b689735269f14 \ + --hash=sha256:9bab2045550c4fe0f7baf89574db1b455c195750702ba96fef1f16972b146617 \ + --hash=sha256:9bf1d5dcc15204d9ec8b8ea4c18fd66e6b80e5de1f4ecbafb3a2f2740f8039d4 \ + --hash=sha256:9c061aa954daaf57d2a4b8374f9fca621ef0e1b603584431c220c22458c59b6d \ + --hash=sha256:9c428eb2bd8817588d16a0ab898aa4eb5d141f896aa2b394cc79a4cf61d9a8e2 \ + --hash=sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820 \ + --hash=sha256:9ce66feae6ac65327379460380549bf1b8df8e17c4e25df2a2bcf168272e3bed \ + --hash=sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd \ + --hash=sha256:a2c473a355f9239efcb72c92d5abfd8fcdb0cc78c8e9af607e72ca12dbb36593 \ + --hash=sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac \ + --hash=sha256:a6d02b4c38de03d9c7617813433e6a0fb6b522797974177d69d9dad431900833 \ + --hash=sha256:a7d470cf7b206e6359fc77b1b860632fde400d5a2ed59cd0181b93a686bc81ee \ + --hash=sha256:a95529a92a446db351675f4aab518feaf5e99842f63f5dd17160c2b74f382db3 \ + --hash=sha256:a9918e58faf62ba2c7147927d06057aec78f42475aff5048047ec47e7265a600 \ + --hash=sha256:a9d3983bd6ab7aa1cfd573544ae98df9b6cb6912a5185a198263e024a636861d \ + --hash=sha256:ab52d8f1fc1b64821c1fbad64a647ed6203627004059a6d1ed4f0858a1499703 \ + --hash=sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764 \ + --hash=sha256:adbeee7d6fd4cf5fe0aece2fb3edc4243615d3180430ba8149d01a90670cac99 \ + --hash=sha256:ae53924aa853a7a2ca20ed4142c7c6b56338e4d4cd999e2980075b9efc2e257a \ + --hash=sha256:b032a0023eb41d768ce77d83210ab2a3c389bc0b09313273c7e1eca48c10a755 \ + --hash=sha256:b1b8ece1e71132d2afba4dbc0c3d62c766e25165990b25db1196c04969eb3d84 \ + --hash=sha256:b2966998927d7bed9db12c0a4647b0c7b179755878fc9c357fe1ffd3e3b0c1a5 \ + --hash=sha256:b3cc509327c7b27f6f4727a8830f4004f6df7766e179f2f4b8e54e65c0bec5d3 \ + --hash=sha256:b7806e804889231b0e06469fd4a5c06313d1c0a3377322b6d9237fa5e0fe4167 \ + --hash=sha256:bea559ad70218d230663e4210875735076a9bfea5994cef34a55a25faeaf2544 \ + --hash=sha256:bf163cc701f3d4ac43ba7d97771bf5fd955220ef5500ef3ee847bc0ecfbf4ec1 \ + --hash=sha256:c147451b4a58e7050f7f7394e6c467867c84161560001f9ad4fb2d1446743946 \ + --hash=sha256:c172db893e516e1358e65a95ee20b7ce7173963eefe318b6ab2a2220688b999e \ + --hash=sha256:c1d60eafd9c7e8e74abd03a5b00df44e7febfe6d9b89b559c0a6551eef0699d4 \ + --hash=sha256:c2c30484dd1417ef98b51021ffa2cc0d7f3c78918adaaaab7e70817335ab3e02 \ + --hash=sha256:c32e26310cc10e547f53cd13d39a369034f69dcb7d749d5cb0e5f67bc196b6ba \ + --hash=sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02 \ + --hash=sha256:c8859a013ae0de1074660992139a1a440df3e6b219b86cf0d3f11c2692bb4fe3 \ + --hash=sha256:c8c4478bef6d57fcfda15dae461ea3c9f06aa7b257c58df3f2300174ccbb185a \ + --hash=sha256:cb11a971a3aea10f9b8373be628f1df932964fc6c6b174516d318a48c3ac4412 \ + --hash=sha256:cb131d775a1573c1aee66656bd78b023577bbdb6cb8349a07773bd4f73e68a6e \ + --hash=sha256:cd88b01f3d37b7a2a34f91d98f14720206f1ea3d540843fab2d649dd5fb91fec \ + --hash=sha256:d05e94cdfe0d15d0206f970722d2554780ce562787b21b218b275447f8751319 \ + --hash=sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2 \ + --hash=sha256:d3112585250b199296c26ca6e0131640b6a8d01bab8b232d2eb3763ed469de11 \ + --hash=sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073 \ + --hash=sha256:d51db97c96384fbfcaf8f4c65922183a68b94f891c3c10c862ef5f6df2adbb1f \ + --hash=sha256:d94e44be379e569758fee8a9a58431cfc3c2598c708b92b1cfe96c66b4c94aef \ + --hash=sha256:dab9ac5a67c8d1f070c00fa8fccb7cbd1b8dcc1a8d6b42f37540df9b3d4cc603 \ + --hash=sha256:dbf53ae2601b7fd5a93c3944deea3a78d40f495226d582c35ef7a433425ce2b2 \ + --hash=sha256:dca3fa8d8a0a26679862eccb0b1a9151b2b9f1cd2c212e7a6335778faaff5833 \ + --hash=sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450 \ + --hash=sha256:e1cc2bfaee8c214f06080a7c7d5772419b8a1108e8e5349236189811823fb02a \ + --hash=sha256:e29347c142cf6e99e0dff5e2995ead1d50fa3b51bf37a7c726a7ccfe5419745a \ + --hash=sha256:e4f5cf4dc72a71c4cfa9751b4950be22f733626670230d46e7d606592aa22d59 \ + --hash=sha256:e724a7b6091f0b1ac064f9d1b15ff9ec52e6033a86cdae649e5f086e32a3c0db \ + --hash=sha256:e87046c8ff77a8decdb6a41d8ab25824b47531b2da933aeab0c1e21c7acff329 \ + --hash=sha256:e95c8def4b81c5d68d5cf1f54c07acd7c0d2577af244e5b6da802120825737c6 \ + --hash=sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1 \ + --hash=sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8 \ + --hash=sha256:ef692a24087a699c0a4a26af45e746e0c1eae2116f6d8a5ff91d8aae2b867b45 \ + --hash=sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b \ + --hash=sha256:f001b571ead90ca1770f1e616db255351a1703317f20374c361ef22f12c06d09 \ + --hash=sha256:f2a7966bda23dd85051f1661ce0ace38d6890e05ec6c357ecae9d2479cba377e \ + --hash=sha256:f2ebb54b3f932210503072f09974b4fb574d823e497a944adfdcd140a6a00255 \ + --hash=sha256:f2ed8b64dc0c651c0f5a9c926777719770021251b8f336d97c4b80b660836ce1 \ + --hash=sha256:f375db73a39f5cf83696d500e21a67f418dc9a988955756f254be8f03b7b3651 \ + --hash=sha256:f59c7673465908cbe506117176156c127f29f917677afceada34957179221d91 \ + --hash=sha256:f8d40ce41991e9d56fab4f5dc4a51fe59bc3b5c77c27f4b148963064d00232e8 \ + --hash=sha256:f9033b43f511f27547c557dcaba0177649e10a3725336ccd2cce0fdc1dc4850d \ + --hash=sha256:fba47bc2c3d7303c3d027c6cf4d07626c37b1314ac81f5820c31032e0ca1f677 \ + --hash=sha256:fbdc5ec49f9ca3cd24955cf3520b10a4d4c901ba2572094c84274e9e7eb30534 \ + --hash=sha256:fce9523df31cea6284f3e2c479876750d7687cf671d7b25d32b19effc0e86441 \ + --hash=sha256:ff75a7537413a86e7cafe98e0e1d6e3dc4b15c6349896e7d5c6b881bfdb6d550 + # via aiobotocore +aioitertools==0.13.0 \ + --hash=sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be \ + --hash=sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c + # via aiobotocore +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via starlette +argon2-cffi==25.1.0 \ + --hash=sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1 \ + --hash=sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741 + # via -r requirements.in +argon2-cffi-bindings==26.1.0 \ + --hash=sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2 \ + --hash=sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e \ + --hash=sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605 \ + --hash=sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a \ + --hash=sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8 \ + --hash=sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4 \ + --hash=sha256:1af817e84578ef8b7295ad17de0f9896e4c8520dbf2233c7aa5aa3d487256fc4 \ + --hash=sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba \ + --hash=sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb \ + --hash=sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2 \ + --hash=sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81 \ + --hash=sha256:242bb0cda2ae3650764fc194593d9ea45fc9e72729acd89778c7cfe184cec2a5 \ + --hash=sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29 \ + --hash=sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31 \ + --hash=sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8 \ + --hash=sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e \ + --hash=sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728 \ + --hash=sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a \ + --hash=sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35 \ + --hash=sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a \ + --hash=sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d \ + --hash=sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca \ + --hash=sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98 \ + --hash=sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1 \ + --hash=sha256:7014ab7e6f5d8511af92544667a0346ea6dfc314ea9a7cad1dba9fdb5c9a6e33 \ + --hash=sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36 \ + --hash=sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69 \ + --hash=sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1 \ + --hash=sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb \ + --hash=sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f \ + --hash=sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083 \ + --hash=sha256:b70225b5fd1e0d2ef4f7fd30d24658454535f0924dff0caca5dc08efbbbadfbb \ + --hash=sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08 \ + --hash=sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6 \ + --hash=sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440 \ + --hash=sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d \ + --hash=sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e \ + --hash=sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210 \ + --hash=sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990 \ + --hash=sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638 \ + --hash=sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4 + # via argon2-cffi +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via aiohttp +botocore==1.43.106 \ + --hash=sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97 \ + --hash=sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78 + # via aiobotocore +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via argon2-cffi-bindings +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via uvicorn +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via uvicorn +httptools==0.9.0 \ + --hash=sha256:02bc5b3dcb6394b9d825fd62a7bfa0b2943063a3c89abc4492ad45e334a20eb5 \ + --hash=sha256:050f7ab098121873c8f13e35857f97ab60a76185c8302bde9a384939bb7c3b96 \ + --hash=sha256:050f84b7ec46a6efe0e5f521cf8729e3397c1cef4384f62ed8d5d68ca0045776 \ + --hash=sha256:06bfe7fad972a417269d8a5fc53b87e4eca970354abf5e9e24336fd06d64292e \ + --hash=sha256:088de1738e1af624466a01c35d652dbe6fb825be887c76d68aa850621d81db88 \ + --hash=sha256:0adc974916efe1fbf89d0363a86dcb2c746727643e362ff398de1a4b50b6bc77 \ + --hash=sha256:0cc339a807c156d840b54f8bf050ba0fc265eb81692c24bca8535b52fbd797c6 \ + --hash=sha256:0fd73d0bbf700a30dd87e4412adf41cfa71542a533d6b390c7244bbb8a1152bb \ + --hash=sha256:130635fea6e611a6b2026120037965ddb88b3dafd11bb64e264b101a70a76630 \ + --hash=sha256:13873eb8aef5972fcfee614f63d47064312ad4efbfe65ade15b8a3b77f8c8659 \ + --hash=sha256:18d800aaa2d6bff7d889df810d1b19a5fde72b1f6c0ca96e8d9f28a692fe5460 \ + --hash=sha256:1a4050a651e1f2faf05eb028ce9f2168abbcee9e24b209f5c1f2eb96d8c569e4 \ + --hash=sha256:1a7f1df31829c258158be01bb04eb668c4fba7df1ddf2262131a972962e651b6 \ + --hash=sha256:1b01c0fcd6725a8d79a164ecdc4116866282479d68bb3d6d74a909bf994656c4 \ + --hash=sha256:1b95775f6292d72cb452c33e5c0f8b8551807c29a10e3c1671fef7f61361370a \ + --hash=sha256:1f6da814aeecbc6cb8872d6d3e85ed16e8ab1653f9557cea8658725ce212348a \ + --hash=sha256:2095207b75a83c9e947346da9c127fb7e4fb29f41589df2643764f06b750989c \ + --hash=sha256:22ab1b10b06d357f01092e60f5e6856a0d479ed79b0ec2166a339ea26c699be2 \ + --hash=sha256:2319858018eedd0c0b2f950a620413c0a9d1352607be4267eb28209eca8b1e3f \ + --hash=sha256:268d18601feb5367885c6ebf6f402c18fc25a324cee215784adafe0a1eef925f \ + --hash=sha256:26e1d9629f3bf70d23f0d22238152aec51c837a7c9e384cb74f356fdccad7eb3 \ + --hash=sha256:272db0c51e8b71e953c1f2ecbe63402b819680e4564be2ef285cfd4584ee8355 \ + --hash=sha256:289f213d2a3dde2e8312c415ffecec5a01698589ec6249ec4e8fb3b47c0444ba \ + --hash=sha256:29b0d823e3c1e7cd1093a5dc889245db693ef13ada624cd66e2262421ef38867 \ + --hash=sha256:310266a2db1377ffae3bdf6556ab4973f4f94508a8ce37b2f6bb096a89bcefa1 \ + --hash=sha256:3238e198429cb8909ec42951b82d6a33fe0fdfcf86371732f8f09311c5b8ac32 \ + --hash=sha256:34266cec8c1d4e3e91fcca7efe38971d6bdda64a7944f2a46ab576da15173680 \ + --hash=sha256:36fac804b8cfd6b935ae64f71349f833d2b6298404626d017a2c57bb942bc643 \ + --hash=sha256:3af4e45ff455fce5511fdf2653c1ce428ef09c56fe37a83eb4d924c2d474f31e \ + --hash=sha256:3e3201fe4d46e0d15d7ff9fafc94a605da9eb82d2c5b9837f0368acb325481f1 \ + --hash=sha256:45b3002392948dcf578029c89f6318e1289a993a1a5ec38a4161560fab60f811 \ + --hash=sha256:465bc1526debf53a3be92022a16ca0c38f891ea3b5c1587af4f52e44020f8a07 \ + --hash=sha256:48c705bd0b1afb6253ed71eca9f9ba7ac7d47838e5fed1ef7891d67f21ecd4de \ + --hash=sha256:4a4d8c2c7e73ba5967be74d7c3a5ff81fde815ee1b48d9c5c0f14de8463a847b \ + --hash=sha256:4a85401b0c3f893cf5695c1199e8679fbf673f7f78c2f6c11d6b1850f8c7e358 \ + --hash=sha256:4c58dc91aefb31adad500aa68054334f429b840b36dd29e34e834101044cb2ef \ + --hash=sha256:4efbee349138a3fee7a4cc3a95abd2d499fae70dd5bff9fed9138d6f570f4283 \ + --hash=sha256:4fb995082fe41ec410b33c48b54fb1d44abb8a6ee762c31e8c42519e8c3a30a9 \ + --hash=sha256:5042aa1c7e2b1a24c17dab31d8770b63a5101c9abc25f832c6aef6b201e1ca4f \ + --hash=sha256:52fe0176682a25b15370f23f5b0f1366a84771df89144fb0cd979cb72a94b5ca \ + --hash=sha256:5332a020a60bbe32ede4bda1a62b3d56c4831d309cdf0932842c0fca8ad6aaa3 \ + --hash=sha256:563e4568217dc907a91843f38c737be865222c0400a38cdcd0d26ce92b3db271 \ + --hash=sha256:581b27663c6e9f4df68068f32fe6d1cd7647b31fac90237221a66f8821c342eb \ + --hash=sha256:58a1b0ec4cbb930e69669f9771715b2c7898d3cdf064d9811f7a66afef96b544 \ + --hash=sha256:5cc5d3a29f9ec86ce406e5ec09c241dd8dc4d30e838f74f68d728b89131a3acf \ + --hash=sha256:63d38e9a9a10a20fb57593742e63c6b1e78dd7f6ef5472de8e0b1e4cf4f3db26 \ + --hash=sha256:6b1ac7f1bc6c0dbf90684b77571a51a21b2463909fd916ce0ac9bfc4d566dc75 \ + --hash=sha256:6b900073e7b8481ef1aaf4f6c1789d210a1db01a9da8789821578cfeb4c2d540 \ + --hash=sha256:6c12d0393a903b58bc5f5a7406d6c5290acfb8284290d68547ce620c06f7d133 \ + --hash=sha256:6e2780e33a58a93f27cc3bb74a55bae6f9a8278a1dbabdff392940d30d381671 \ + --hash=sha256:6ebd39ee26db460cfe5ab8b71a15d1149b289139a0d3981522757d6af620887e \ + --hash=sha256:6f8b41299b203ce8f627db670cfea82067d9638853dbeaf86dccd93878879b85 \ + --hash=sha256:6f9549ca354a1d6d6167c458a1f1b12147726b968f02dd64b6a5801dba91ae0f \ + --hash=sha256:6ff0145b34610e57c9fae20df4e133c8d54266447387de6fcc0bdabfe4db4569 \ + --hash=sha256:6ff5f0ed70783dcb9562dbd20edca51c3d4d277f128223709e3da6b75986d1d4 \ + --hash=sha256:714bf348f468532d86bed670837e7d5ddff3834dd7f5d3c08066da400c86f088 \ + --hash=sha256:757e3f79cb865a7db94e0db5f4d0ed3284a69e39d53568f433982ea13c60cac1 \ + --hash=sha256:7e32b83bd8c2f8b6fa726ef34e63e21c4d7eddc277d40d4ef7245ea3ed28e5b6 \ + --hash=sha256:805b0f2618e5d4c3e28f45b731eb1a0539691ae4a2f97b4ce014de0bf96a1ff5 \ + --hash=sha256:80eae881cfb69383303e9a4d7961a478025b89c24f38f2e69b30c516fa0d57f2 \ + --hash=sha256:813a32f94991b9627795528053c73a57d2ce3eb98ede89f0e1c7a31095938e81 \ + --hash=sha256:8463b34ebde3f000627e9dbd8a545f995ad49fbf7ff9dd5abc0cd507da98a603 \ + --hash=sha256:8a59c749a73fbdbc8e63b895a3079825fa085d752e75bc0a500042cb8a801e48 \ + --hash=sha256:8d90d10e9b6594c28f27896a68fab97fd784c43804e9fe419dab8e8dcfcf4b02 \ + --hash=sha256:8e1e037bb57dbc549c6fe20370b763ea74bdb09413cdcf857e4f14d9e4e2fb13 \ + --hash=sha256:931f45f84e15daafec5f82cc92e6710569e1f50933f3253d206eab4132bec678 \ + --hash=sha256:995b52f7c260ac7023640221f27472303968753cb6fc6fce1ddfb0e9db59a398 \ + --hash=sha256:9b4da5789d7cf576c7e81f0088c632f6ee3786d87d17f08e90e703c22ce15633 \ + --hash=sha256:a3ed60ea9a7c352c590182c67404599e6b5a0c901e75ae4cceee9a9fd6bfa455 \ + --hash=sha256:a4d1ecad62e83cc65b411ea0125972cf3af98821e8117129947fd1e3a113f8d2 \ + --hash=sha256:ae9bb62a7902e2ab65782447cd3eeb753510feace4e3ea03937a85489b01b16b \ + --hash=sha256:b2ab3aad55d75d0b8df8d8a1b5920baaec9b161112cd5e95984848b4d2cd3dfe \ + --hash=sha256:b2cc6991f16f6d666d48e4b57318104e7b29109e32e2f6b86e9d44c4e6a27f4e \ + --hash=sha256:b5a3f5f70967a1aa2bc47fec42a1e19d2fb38c61700e3ee62b63a4af4f4fd001 \ + --hash=sha256:b68fb053b37c258a473ab67f4965c3b439500dc160fe364667035a6833eaf50a \ + --hash=sha256:b6ee42112d785a913dd63ec0335435a3dddbea5040c151252db815b0095cf066 \ + --hash=sha256:b928ab0ecaa664e8caecc529dcb8bc881b6b35bb2b74bf9a39ae25f982ee8812 \ + --hash=sha256:b9430f65db521db7962ad951571d446171213686f96c998a54dc18ed574821e2 \ + --hash=sha256:b9cd15cb7cf0d5cc41f649fd789aae12c56c3b83eff593f8e095c1d4555ad5c3 \ + --hash=sha256:bb1533541c729ad422f870a780d8b4af924f9817d45b5f580390418cda72eaa2 \ + --hash=sha256:bbf7377fbd41b7c87d47820e25b9876724963681c2a1d6f6ff2adb4db46ac174 \ + --hash=sha256:bca180cbe84e4fba7807eb408a8655295f697928512324517e30a091ede522a8 \ + --hash=sha256:beb2c8a34cc90fb4d862b7284eafdb322030d6a8b2ee5eb6a744f84205beedc3 \ + --hash=sha256:bfdabac0c6d3d6a5be8c2a100a001c92c14a39bbafd5999545a675c493626e64 \ + --hash=sha256:c0e45def4d9ce7073e2226535572442d9d6efb4047c7a5fd8960807e877ce70a \ + --hash=sha256:c0f537e5e8152e8d9cae82804024790cb973061abd3b7ef8f66f46e2b5c7bb51 \ + --hash=sha256:c195a69df0ab2541252ab5b1d76e3c182e5688ac2a9b708e5e6f66aaeda91e9a \ + --hash=sha256:c271bfb832be5c5c020b4e2fcbc1e70a0b990adba6de874b0bba1184b89cdea3 \ + --hash=sha256:c42424213c28804f8d0e20f5692106cfb57bf72e1dbc4092b8481fb2f9e4c707 \ + --hash=sha256:c4fa57d3c31889722f64bfa785545a5e603a893b6f29ac1a41bfa830abeaefd5 \ + --hash=sha256:cb2bb3ac0af7fdab2311b895c9eb95442b45deb14cc949b9e65545e74aa0be69 \ + --hash=sha256:cb3e7a4fd0168e362673a980380bf4fd6ae3b1555150e60c5390b4b10d9c50c4 \ + --hash=sha256:cbbfcd5d15056fbd1edd5e725cf3feeb47c7cbccbe205927ebab422cc229f417 \ + --hash=sha256:cd3e55223a77d6e08d5730ebacb4930ecca5d2ce7c57e7ba10833be7e52903f1 \ + --hash=sha256:ce8e723b4637034b76f5382a30a6b725518c332273e8d62a6c7d46e90837c947 \ + --hash=sha256:d1e329a1866981efe0201d05a374617f6c6cf14434a501d78ab22793d1ab1fa6 \ + --hash=sha256:d20ba5c84cf0592afb2713336f07e2b6ced082e4ae803ceada153a85613efc9f \ + --hash=sha256:d2b095129b9a98eb46a271ee9631089529c4e40354576b4aa74e24de9d2bf2f7 \ + --hash=sha256:d3906b5c549ff2ad2473cb711e1fc65d76715c2726a402108fbf55eab6c6b49d \ + --hash=sha256:d484ebb7e3a3f3597b0f645fbd1b85633674ca808c1f5ba11c2caf7c66f5c8b6 \ + --hash=sha256:db735a23ecb0f0450d2b24e0a05fb00a8a35c9db172919c4d3e023e7c7ee4c9b \ + --hash=sha256:dbc9fd1521e573045d71b6afab7398439c5cc259e8cb9d416fe62d485c4899c6 \ + --hash=sha256:df3867518b205be3648e2fbd522bf380c851b5c2500588047505afdd786b6669 \ + --hash=sha256:e0acbd474d0af4afacc6e66c4273f8a19e25f8af4379fc816388095ea6b01371 \ + --hash=sha256:eacf0f45ca3ff84c01481c60c15da9ee56711f7292f66663df0f57af61e011c2 \ + --hash=sha256:ead1a40543a033a6732a9e1e515944979a19db3737ce77363fc0660e38554344 \ + --hash=sha256:eae4e9c7a0785a1a715de0a74fb822ab40084c060f444f18f075d05e322aa7ef \ + --hash=sha256:ecf7037e491c220cd73987838c1ac3958d787bb098c3be0bfaf7f04204a6162c \ + --hash=sha256:ecfeee649184ffd800955068be9a6b579a0f33fc3c98535d685d5779cb59347f \ + --hash=sha256:edd5aa045fa3cc57143db018dd32ce7962bd5b525d05230709015d7e570100aa \ + --hash=sha256:f0ef48ce353f6b6a52232ba23d0983d4c2c84c84a778899404e34b4718509bf2 \ + --hash=sha256:f1734bd6f588975ffc246211e8b96c11933344087ca280d2cbcbf35cf835d7a9 \ + --hash=sha256:f67db0ba2bedafec15b8e5330d40da1e1c7921559fa715af021252bfef81a6f8 \ + --hash=sha256:f6ac1414556b910a879c108d79736f77e797871f9919ed0d2c3cf8cf3ecca986 \ + --hash=sha256:f78f7ae1c2e5aabf29583fc0d302d8081a663776f84578025662eb6f5d63a921 \ + --hash=sha256:f9489c1d87160c126f73b004742fe8654fa1ce37ed89e9e01330a1c10aaecde4 \ + --hash=sha256:f9ccc9884241efceb4547a92955d128574c864681f11b7ea3ecbde295fafbe8b \ + --hash=sha256:fc1a4f9d18d32a6e0a0a0a382986a60a2126f5144dd08715be7adb8df18e8a46 + # via -r requirements.in +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # yarl +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # aiobotocore + # botocore +multidict==6.9.1 \ + --hash=sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c \ + --hash=sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9 \ + --hash=sha256:02f6d0c4b70f783305e73f9944d8efe6be1022550f0974ba0ae9d8893c0350fa \ + --hash=sha256:02fe09dc197b8ae7e355371e51e5dce2f39060cd5a8badf94904527e23a3f188 \ + --hash=sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201 \ + --hash=sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382 \ + --hash=sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f \ + --hash=sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919 \ + --hash=sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418 \ + --hash=sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771 \ + --hash=sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c \ + --hash=sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76 \ + --hash=sha256:0dd655518f136febd96c05131a76a863e32fc2a1d7acd4e3c959e3ceb77d8345 \ + --hash=sha256:0f06e60fa190aa7abd0914c2a766736fdc8e9f34878c4346338534b73d1b20e2 \ + --hash=sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580 \ + --hash=sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9 \ + --hash=sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1 \ + --hash=sha256:13849a1d4f54c3809ae721e9e83ab28f5ea602f33660cb84eb6ef261eac706c1 \ + --hash=sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64 \ + --hash=sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5 \ + --hash=sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006 \ + --hash=sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8 \ + --hash=sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02 \ + --hash=sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132 \ + --hash=sha256:1c9f3c25df6c9d3bbae4f6fd3f514b1c5c740a2110f56ccf36069c85417e289c \ + --hash=sha256:1ca5ebe6d454f1e5496cf052386a559f1080bf2de75bf327ebca0a6003b79f19 \ + --hash=sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b \ + --hash=sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba \ + --hash=sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b \ + --hash=sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08 \ + --hash=sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe \ + --hash=sha256:2c1aeb92eea59d824f004341b26d5e4b47a8a441cf9726769b9a90abf9d0e08f \ + --hash=sha256:2c5d675da8f1cb5650271c8ad5e95c0a3e5a183c105e72d953b12877b1c8d0fd \ + --hash=sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879 \ + --hash=sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f \ + --hash=sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9 \ + --hash=sha256:33fa55b990f81c2927e01399ace0d18926c69d69baa8cdaa819424132fb97987 \ + --hash=sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd \ + --hash=sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea \ + --hash=sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc \ + --hash=sha256:369b5aa01b241cd3fea6890bdbb11a1425d87bf1515831500d518f4223e9d72c \ + --hash=sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b \ + --hash=sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709 \ + --hash=sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4 \ + --hash=sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce \ + --hash=sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94 \ + --hash=sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a \ + --hash=sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e \ + --hash=sha256:43124fe172ada86d03ac3c8dc8179091341f6724d5e5d5b160e1587e4cd3761b \ + --hash=sha256:4736d350371337825cac1793c9f7c40701c32a03912548c2a7608e51679cbb96 \ + --hash=sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1 \ + --hash=sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e \ + --hash=sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455 \ + --hash=sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3 \ + --hash=sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471 \ + --hash=sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d \ + --hash=sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b \ + --hash=sha256:5800368526647146978389dfaa46da3356291e9f0fff9a4ef12e8c2bef964a0d \ + --hash=sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f \ + --hash=sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004 \ + --hash=sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4 \ + --hash=sha256:5f89dad732280e7a10b74d40b91364f88e13c3f2c08c2ef83a8cd42f7a61af2e \ + --hash=sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5 \ + --hash=sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60 \ + --hash=sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14 \ + --hash=sha256:658f90f49cf5af2441cad0a2b801c3ef520471989a1ec55bcb25b255b2ca8d2f \ + --hash=sha256:66987aa68b0f7c2a1cc5f388ca962b8ed92b10f79de38d6d0d8c716154f519d9 \ + --hash=sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3 \ + --hash=sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194 \ + --hash=sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a \ + --hash=sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3 \ + --hash=sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42 \ + --hash=sha256:73533644e1f69ea1164d56cc6505f564c6c44072b646b66d67df2d31fed0348c \ + --hash=sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828 \ + --hash=sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290 \ + --hash=sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485 \ + --hash=sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e \ + --hash=sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8 \ + --hash=sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c \ + --hash=sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b \ + --hash=sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659 \ + --hash=sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b \ + --hash=sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9 \ + --hash=sha256:7fc59e9ba821b220944ccfe0f89c9dc4745f6d097569992356eb03869f21e953 \ + --hash=sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c \ + --hash=sha256:803f8b575a71b1b299d677c28db0653459c79b5308874efec813f17b7457c7f1 \ + --hash=sha256:8050e75af7e4c6e2d5260b84eeedb618f3e452e66473432e085b5d1b81429299 \ + --hash=sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94 \ + --hash=sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983 \ + --hash=sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d \ + --hash=sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac \ + --hash=sha256:87cc632c88ee5dc80e12681047839304d98ee5c9a708d686505767001c9b8b9a \ + --hash=sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7 \ + --hash=sha256:8885e3808aedbd6725b921fb67dacaae0678933561ddd47c2b01315198c70e2e \ + --hash=sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45 \ + --hash=sha256:8ed78ccad4c7b421804f5d524b7740946a7ef75d89dc0dac52e9d7c24c472410 \ + --hash=sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40 \ + --hash=sha256:910d4260512660484c0dc1588a316fbb35a40c081c36fc51d1225351af17cfe4 \ + --hash=sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805 \ + --hash=sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d \ + --hash=sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa \ + --hash=sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d \ + --hash=sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701 \ + --hash=sha256:9ae9614c317c50836689ce2dfde07c05fe0b16378562e2221746c3913ede3c80 \ + --hash=sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6 \ + --hash=sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40 \ + --hash=sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20 \ + --hash=sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694 \ + --hash=sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a \ + --hash=sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203 \ + --hash=sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519 \ + --hash=sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125 \ + --hash=sha256:a32b78c1e52ebd8e247bb68300b90b233300d8816faa008ed0713bc539fb6af0 \ + --hash=sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d \ + --hash=sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a \ + --hash=sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea \ + --hash=sha256:ab64ace1a68682d191d9bedd9d4c939406ad86b1d9f628180410644249ad46c2 \ + --hash=sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c \ + --hash=sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3 \ + --hash=sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d \ + --hash=sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3 \ + --hash=sha256:ad4528cdce058b684f75fad1faf4a6a6c992fe2f08376370ca66e4ce5916a84a \ + --hash=sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68 \ + --hash=sha256:aef74e9beabbd6c4aafc091dabff86d046ccf013ce1e4396c0fbb01b4cad9de8 \ + --hash=sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca \ + --hash=sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8 \ + --hash=sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840 \ + --hash=sha256:b2f0adc22a4eb31e545221d93fc73a0f6a8cc2379d0f4309f71d1d17ba938b82 \ + --hash=sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610 \ + --hash=sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca \ + --hash=sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a \ + --hash=sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1 \ + --hash=sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5 \ + --hash=sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a \ + --hash=sha256:b828cf64d62dc09ac183f03c1aeedd164ade96a2ce4934109452edf29de1dd13 \ + --hash=sha256:b8a65621b98984a62e59403009591b8a5a7736273aefe1cab64cfb85b365cc07 \ + --hash=sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05 \ + --hash=sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1 \ + --hash=sha256:bc94a68ea5e18f8e85dc6b522bcb53093f692c8eb62b4837ac047da73956cbe4 \ + --hash=sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2 \ + --hash=sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82 \ + --hash=sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6 \ + --hash=sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d \ + --hash=sha256:c9648ed33dc8179e4ec04bbc73bd7f0038e1e81217a69467f61f02a78bf07e88 \ + --hash=sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f \ + --hash=sha256:cf22b43f35b7dbb9f71e8ee2041b5c00029cdfc28ede3a2f31cf8906f9a6c126 \ + --hash=sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24 \ + --hash=sha256:d3b6e6840421c83ccb60398e333b44f910b0907bb409597685ab2eedd1e22eab \ + --hash=sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67 \ + --hash=sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3 \ + --hash=sha256:d9d6544790ba50438a9c1a903c3c4afb1ec8a7832db5518549275b40ef0dd4b1 \ + --hash=sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800 \ + --hash=sha256:dd9a137a4a9becda3094f3831cd026380f75f6855e051eefe4c73ade524f1cc3 \ + --hash=sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8 \ + --hash=sha256:e3b1aa25f01238886a6baed9e13b9a9240ed344346c79ae280ae65e8603b21d5 \ + --hash=sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24 \ + --hash=sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c \ + --hash=sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a \ + --hash=sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841 \ + --hash=sha256:e8f1e362c9352b50ed120f001046fdbb80810c9d56580f4c3fc13bbe30823387 \ + --hash=sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec \ + --hash=sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721 \ + --hash=sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d \ + --hash=sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390 \ + --hash=sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1 \ + --hash=sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d \ + --hash=sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e \ + --hash=sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df \ + --hash=sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb \ + --hash=sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5 \ + --hash=sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9 + # via + # aiobotocore + # aiohttp + # yarl +propcache==0.5.4 \ + --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ + --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ + --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ + --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ + --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ + --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ + --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ + --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ + --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ + --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ + --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ + --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ + --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ + --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ + --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ + --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ + --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ + --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ + --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ + --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ + --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ + --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ + --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ + --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ + --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ + --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ + --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ + --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ + --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ + --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ + --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ + --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ + --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ + --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ + --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ + --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ + --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ + --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ + --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ + --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ + --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ + --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ + --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ + --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ + --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ + --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ + --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ + --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ + --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ + --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ + --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ + --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ + --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ + --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ + --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ + --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ + --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ + --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ + --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ + --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ + --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ + --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ + --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ + --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ + --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ + --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ + --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ + --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ + --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ + --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ + --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ + --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ + --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ + --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ + --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ + --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ + --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ + --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ + --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ + --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ + --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ + --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ + --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ + --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ + --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ + --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ + --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ + --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ + --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ + --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ + --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ + --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ + --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ + --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ + --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ + --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ + --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ + --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ + --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ + --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ + --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ + --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ + --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ + --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ + --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ + --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ + --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ + --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ + --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ + --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ + --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ + --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ + --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ + --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ + --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ + --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ + --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ + --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ + --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ + --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ + --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ + --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ + --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ + --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ + --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ + --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ + --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ + --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ + --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ + --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ + --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ + --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ + --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ + --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ + --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ + --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ + --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 + # via + # aiohttp + # yarl +psycopg==3.3.6 \ + --hash=sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631 \ + --hash=sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2 + # via -r requirements.in +psycopg-binary==3.3.6 \ + --hash=sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781 \ + --hash=sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2 \ + --hash=sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475 \ + --hash=sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372 \ + --hash=sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de \ + --hash=sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03 \ + --hash=sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840 \ + --hash=sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79 \ + --hash=sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b \ + --hash=sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e \ + --hash=sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5 \ + --hash=sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9 \ + --hash=sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f \ + --hash=sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe \ + --hash=sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7 \ + --hash=sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138 \ + --hash=sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf \ + --hash=sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d \ + --hash=sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a \ + --hash=sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f \ + --hash=sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4 \ + --hash=sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6 \ + --hash=sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2 \ + --hash=sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300 \ + --hash=sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0 \ + --hash=sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a \ + --hash=sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6 \ + --hash=sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7 \ + --hash=sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc \ + --hash=sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e \ + --hash=sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30 \ + --hash=sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba \ + --hash=sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2 \ + --hash=sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22 \ + --hash=sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef \ + --hash=sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e \ + --hash=sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f \ + --hash=sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c \ + --hash=sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c \ + --hash=sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299 \ + --hash=sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e \ + --hash=sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638 \ + --hash=sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba \ + --hash=sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a \ + --hash=sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9 \ + --hash=sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc \ + --hash=sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2 \ + --hash=sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874 \ + --hash=sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c \ + --hash=sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e \ + --hash=sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312 \ + --hash=sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8 \ + --hash=sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac \ + --hash=sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18 \ + --hash=sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269 \ + --hash=sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb \ + --hash=sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10 \ + --hash=sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f \ + --hash=sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1 \ + --hash=sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784 \ + --hash=sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492 \ + --hash=sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc \ + --hash=sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52 \ + --hash=sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff \ + --hash=sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4 \ + --hash=sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8 + # via -r requirements.in +psycopg-pool==3.3.3 \ + --hash=sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37 \ + --hash=sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d + # via -r requirements.in +pycparser==3.11 \ + --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 \ + --hash=sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc + # via cffi +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via + # aiobotocore + # botocore +redis==8.1.0 \ + --hash=sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25 \ + --hash=sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb + # via -r requirements.in +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +starlette==1.7.0 \ + --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ + --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d + # via -r requirements.in +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # psycopg-pool +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via botocore +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 + # via -r requirements.in +uvloop==0.23.0 ; sys_platform != "win32" \ + --hash=sha256:0305871ac712f54b62af73f943dbf21ae3ce80a44bc0f0151424484affa85645 \ + --hash=sha256:090865d8ce7a03986755a3ce711b7dd0d4b44eb14ab74368b717f3fad1180208 \ + --hash=sha256:098a85e1393ef5202767b7e5fb41a32cd8bd81e6ee4af364c179801c4aa3f6d4 \ + --hash=sha256:0efdd55bddbd36bb2fcb842d64c0d5f6407c6958c68088cc25df8c09edc5b5fd \ + --hash=sha256:12634f15e6625f78b3f2922f91404c4d7173487eba11746764153f556e9852dc \ + --hash=sha256:1748321e3c59a14a75404b1ae8d5a8d81c4e201803ea0e14c1b6fd84421024b5 \ + --hash=sha256:19c64108b507cd0bc140e400e3396bacebd9d504956aa7726272bf6de7d9aabb \ + --hash=sha256:1e84575f11873c109cf3962ad0bdf679094466184125f4cadcc41a73febff41f \ + --hash=sha256:24c58ae4a83e93a04c504bcc678125e36a0bfc44af928ad69444880c60f187a5 \ + --hash=sha256:28d160f51ab4da3b187063652e643dea6831072add4adc1e6d62afbe73b6be27 \ + --hash=sha256:2dcff2d69be43e6559e5dad2c5a7a2dbfb60e05a77311b6c4b7a4a8123d86c65 \ + --hash=sha256:31e0cf90bc8fd88784f6802cdba968a51fb1aec1cc3feec74d862b2d371d1330 \ + --hash=sha256:378188efbb1524f2219d05246a3e1e5907217848d2882144dff59585f1b81d55 \ + --hash=sha256:42feced24b9b44b856c633eafb5cc5dec354972da55ce77598db6844c054bc7c \ + --hash=sha256:4448e9124537620f9c25d004c227bb5104440b58955c19bbd312d910af919a63 \ + --hash=sha256:4a08875543bbd4519faf30497506c9cda8a48470467ffdf967c7313c7a5981a8 \ + --hash=sha256:4b8e207c67d207a8608fec57e116511030af3495dc0109b8c333cf9cb412b16f \ + --hash=sha256:4bb7f5d0b62b5afaaaea2b7b60d508921c24b0fe39c22c1438bec1811ffe10ec \ + --hash=sha256:4f1798f56c6f4ba5ac11fa2869e5717926e4470d97a1dd42b4f59219d43b5027 \ + --hash=sha256:514698d3683189031dcbfdc31e87115992e5ce9e1b19fe5359941323f2df800c \ + --hash=sha256:53c2c5d7e2024e46776c2d90e6c637d01102126b61aaf5faa5edaf05f8b5722a \ + --hash=sha256:55d6f4135d914305929fe9e9c44d8b5383a9b3fa1bee3bfcf60ee97e01af07ea \ + --hash=sha256:5a2bbad3a63007f7e9524d4903ba04fee252557c2acd86f9a3d4f91786695254 \ + --hash=sha256:5a3e0f56ec19bfd9ad1605572878dd6ff7f01b325f4fc154812ae70d615c3aff \ + --hash=sha256:5bb9be71d9ee39b4359b832f9569518ec9bc08704194034e79e4958e6bc4d46d \ + --hash=sha256:60ec798c40a1810d282ee046f61ecac1c5675cb898763d9f08d97d53a5e00a81 \ + --hash=sha256:6b3cbc4f96ddfa1fb88a78a69dd851369825b7816d9702eee8c4461505ba172e \ + --hash=sha256:6c7ef4701a96553514b2688e342ef1bf2beae6cfd172d89a76c768292aabf405 \ + --hash=sha256:7337b06a9f9ed9ea3049f04b76f65819db9b19bb832ee598e97b388eadf25e5f \ + --hash=sha256:76345f51367fb1f23e08605c6efb18374f669be5b223658fbab6b17627950507 \ + --hash=sha256:7e35c9bc977760981693e1a7a51493b58ee5a501f9ebb1e547565ee40b6c6208 \ + --hash=sha256:80cac5cb90ed7b9b72a217a1d6982b15b829cdbd0ee6bc19b93e3a9e47fb0ac9 \ + --hash=sha256:8af88fe5c7dd68fe1fec6dea8155caa1a47155d219a750ff34049541cf536a5e \ + --hash=sha256:8fcd721113260ffb5e38bf14a8725b17d431f34209f7d1c7005b667946e630b3 \ + --hash=sha256:93087a845cdfb35753e539354ac9551bdd2ff528c202a98df0ae46e852bcf021 \ + --hash=sha256:93935ab27b6eaef4c3e5489aebc84284f0644592f7ab516df60ee1b27eaf5eb3 \ + --hash=sha256:9bf08e4b6362dd1c08623bbfa2d061e8bac0f1da8fc2007062cfe1dc360a49fa \ + --hash=sha256:a6ac96da66c35bf789bdcde78a88dc7d56b7907d8379648c54adc1c61594575d \ + --hash=sha256:ab17b3a8aa754be0de0e397f7b95f13b14e56f077a4c6ae295e3d4afd199b325 \ + --hash=sha256:b0d106d9314546d69b3df1b5352639aa628530ec3ecef8a98a21942d2a2a64f5 \ + --hash=sha256:b90397a50ad6332ed3e459c648ac20d182cce24a557354363ad85fc9ea4a17cd \ + --hash=sha256:bbbdb8fcd5e7062e546eec1ac78c28bb21ae7df54c18f8e4b06e15a18d661a49 \ + --hash=sha256:bd6f2f81c7b9da99d301c0b16b82044e76fe887086e42e1590ecf520b94dbdac \ + --hash=sha256:be53e1d5f83de43dc175c87612ecc128d444b38e5c56cb3f807f5a73d6887476 \ + --hash=sha256:c3f23f403a273900d57de6ee5ca0614c650f7f58563065dad1a4744498960e53 \ + --hash=sha256:cbe8d03d4efcccdb7fcedecbaa1e1fa02913eaf3a74cb933634a6bc6d2ea9e2a \ + --hash=sha256:ce17bc317d089f361b33521654c13e30eacfd3d2034fd34e613ca9c51c969686 \ + --hash=sha256:d918d6f304a309222a784bbd140b85ec5594d97e4dc0e79f590549d28970663a \ + --hash=sha256:dc61e4f9e37b507069dc7e659ae28bca7adcb04c993c3508214315d12c63f848 \ + --hash=sha256:e095f9e105af76593b4c183bb0bcbdae64bd913a59ec595732dc108b48730ab5 \ + --hash=sha256:e2cba180d6451822763eda8364f342435a873bcfb3849cbd82fdeca248ca65eb \ + --hash=sha256:e49eba8f1e28e7c03648b7a476e1ba05309e087ccdea859fc6dd659564aa8d7e \ + --hash=sha256:f1341c6abcee1c31277cfe28d34e46196f2143ec3d755e6efe7452126e1f626d \ + --hash=sha256:f3fbfe82829d8e381426a289b87e59e585278728361db9ce975b88b51f64f410 \ + --hash=sha256:f50b580fad005a092ed87c5a3a4683459b21d1620497d6a5bccad203bee4c071 \ + --hash=sha256:f5576e8ae1723ece60d8f93c6710abf784714e99388bcf023ba9ca800bc587f6 \ + --hash=sha256:f673d835bdb1a60229cc3609a113fd2c9ce3f4a3c75ad4eaed111180c00199d2 \ + --hash=sha256:f7548ede3ee908cfabc0d068106e303a9a2d811af959cdf6ab85676344cedcda \ + --hash=sha256:fa8ed556fcc87a4091cf61587ef172fa104323dc89ecc085a618ba7ff8629a8f \ + --hash=sha256:fefea5cf8cdda9053b962ca8a90216fb0b1d40907dcb6819382b42e483e6e9f6 \ + --hash=sha256:ff7144d8167e513fe39fbb46bffb4f6f192dfb1f4b0b4e9102e1fd4f212e4747 + # via -r requirements.in +wrapt==2.5.1 \ + --hash=sha256:016602dd8827d190280a707c5e67f9a80038f54bac1782cc8ff68a2a16c618bc \ + --hash=sha256:03aa7d2256309b57ddbf317bff2cae5f47e50ea9ae8d582780ebe0b554347b42 \ + --hash=sha256:051220e5071fdfb1a6678707c8abb7bbf4824d40f99758394b2b4d64855fb284 \ + --hash=sha256:0591e6eace0d186c9ef1ecd1244be5a04e98041424cfca425b684ffe4f0d8030 \ + --hash=sha256:05f6138d5833edf68d88f950ea71bd96daf0a9505b53abd48aa002a0b6d05765 \ + --hash=sha256:06740dbf984af8a26d4b63b75a6ee4e88846c068dc865486ad906448079f50d4 \ + --hash=sha256:094b847491b813b6e6c1775e03770930d75078c0821adf929ac712830951ef25 \ + --hash=sha256:09b1893ee4063706574c1813abf479b8b51926633fbdb6f96aab8dc7b0976668 \ + --hash=sha256:0a526227efe17dd94bd16b123d170f879bce42c15f10eb92495a745f54caa943 \ + --hash=sha256:0c9480bdee340a1602cae5a777146ab4be3e384fdcb569fffdf8721032314645 \ + --hash=sha256:129cab3c7b21e68e693c2819a95c47f3b1c41a834b931154688c83b6aef6bdab \ + --hash=sha256:12bee472452019706fa1d4ead093f52a9683b4fe6617953e15bab9acdfdc013f \ + --hash=sha256:12d3d2b9d6553df6e2421ab99e1cc5413509076788f57fcb3169f5ce100a19d1 \ + --hash=sha256:1425fcf0e70b27053bd610d57bae975856e7897e3f6ba1456d2b80b9d7fd15d1 \ + --hash=sha256:183bf0bb893f783c9d22f953cb01fababb9f618e098763f8e66337b575b0647a \ + --hash=sha256:1910be5adc0232cc6e8c0673bf3f41c2ee724547543526bed8d00734458e7bc5 \ + --hash=sha256:1a96e2671c60f9f09ae547b5a815cecb29af16caa68d73693387d0028788cb32 \ + --hash=sha256:22300c5f254627f24ad2197998fde26db6eacbb0f879162944bf7bd79dd5ee5b \ + --hash=sha256:22a9fda6ac53536ec74e3e334f3568af2535a3df1ae70e8f2816f77160c386d9 \ + --hash=sha256:25eb4d928a9abeaf70ca786a35861b46d1ab37cc4ce49ea70a070dacdead4dfe \ + --hash=sha256:25ed8b1b39234140d5b5c6a273130c7595e0abece417c3ca3cb378fcea5cd0fe \ + --hash=sha256:26313f38d18d40a9975123a4ebff9da125ec63ab9ece4f05320a3d8d37d2c1fe \ + --hash=sha256:26d8ea2ec6818aeb656bd8a9e745a6f1fb0edfcd8f54291ccd94f62eb5f5e3bd \ + --hash=sha256:29b62e87fcd6a1893f669abfd02a596a7fc5cfa79fa57e42c4e650a6c170c67b \ + --hash=sha256:2c642a83b6703804b571caa3b8b205aacd341b1b37e2b2d89cd70e03e0e9caa6 \ + --hash=sha256:36d7d0ad593c4f1a651e4032de834db59aee1a929ee396cd483895b673328e51 \ + --hash=sha256:380f72610181883f66b41442cfc7c0f7552b42169efb2113def26e6380013d37 \ + --hash=sha256:3cf273b7e8d2038abb7f0a8c6550aff4f617b9d486a9965c8e8acc96a3a04de9 \ + --hash=sha256:3f93ceb0ac4896de45d5a45a8f4e69474da583440589de10b362ddc1db4691ed \ + --hash=sha256:4b3f410c416752e1dba53d361e2e6562f22c2c3ec855740dfa5836e061b22571 \ + --hash=sha256:521bd5ef2a33171fac08a0a302d51a983c19c3519406c1ee8da7ce29285488da \ + --hash=sha256:5ad562c23e61e626f9d27aa37aa5679f1c29085de1f998466d107854048bba9e \ + --hash=sha256:5b53000b424dc2133eaaf22838a2352d3497f5d7c2e7d9a2acfe675ab7225bb1 \ + --hash=sha256:5be9816d9de88f02fce23cf55f392403411d9bd9c7ae57fdc965a43b22e2de5e \ + --hash=sha256:6201c7e122f40060a9b50696d80deec8f93b1a235ec0443f51d7a8a42f7044a6 \ + --hash=sha256:6405ff2160af9d59132ebb076eda0304db44d9d09809582932412ef7c0788a36 \ + --hash=sha256:69fd0fbb3daf7c8c6f5e062847a0061f880f347374d74cf1daba57220fb64cd0 \ + --hash=sha256:6e3eff05ae616671b40d7ad0a504210329e4adc9fb91415663570aca93c5f5cc \ + --hash=sha256:711e73da3d7983547fc9dd208973b6b0c52640822f5d477910ba24622df6ba64 \ + --hash=sha256:729d644b6acaf4846a4ef81b037857b66a01dea6d227f827c6d71c0b6d656d6c \ + --hash=sha256:736c1de0230c6d24327b14684794214167b2c5ebb6332e28a10f504641b600df \ + --hash=sha256:76f230a9b07e3cb66646d265398f579abb6128b1bb4cb97c74b1ae5d09e96f31 \ + --hash=sha256:7fa321270b40f3e8cdfd954b3a8dcafc6db1d8bbd4d681b92dfa6b9ef91a9a99 \ + --hash=sha256:8078186f719a92693199f1e06c4ec72e1e6d374c2e459da18ed5c39d6966d727 \ + --hash=sha256:859f67bfc31eb7ab55f237b629cd4ab0441b075912446481f910f7d02066811e \ + --hash=sha256:8922821f66ec08a39f72247776c6158db5bfaa09d0c8f607cd854bdf6b2a2c10 \ + --hash=sha256:89d9a8607b7028054bb6fd01d437f205534a5d59d53c3665d15949a99a2fce0d \ + --hash=sha256:8a7c078323e6e1534968cb85488c5eb7ee2b9bbd0f8a291095213a763da40dab \ + --hash=sha256:8bdf4696fb5bb141a7f96710ac6d9a6aa9a57a14c54075f9c7d3946869d457df \ + --hash=sha256:920f700ef41ee774a1e4778c1f4295e117f1ff3435a7e0cd3e997d10da819d32 \ + --hash=sha256:9a34640eb6295f33ca23462977de275fe8f3a50ab339b8918b96d69a7451e2e1 \ + --hash=sha256:9aa7660684d73925c0d1e4f8536ccbaf233cef3897e33a8c2ec462f83b338323 \ + --hash=sha256:9bad4dbb4e61624fcce5f301e37f9e743ecae4f1259a3777b3207eb7eba3dccd \ + --hash=sha256:9bc472825027b276d4bf678d2ac64149db0b122f80ae6f59c423e6d31f0c4bb7 \ + --hash=sha256:9f0750cbc2e29e4f3c9529d3587d4e7ed8f60638ceafb80b87a95833b0c5acd9 \ + --hash=sha256:9f437dd704abc4ee1bd03bb2d796d362d0e75915e8f3113a7900b3b7ec5f8b47 \ + --hash=sha256:a18e63910252eb75d8806b4baefbc3a03612502f63eab042e3741b00b719f043 \ + --hash=sha256:a1e823aecb3746b8f9e0aee2e1413887871ee2f5c502a3e0ef8d466dbd4adde1 \ + --hash=sha256:a424e8a9776c06aef6313af1d0e3fe6e0838af4241d0c09eb0a3b46f2c9a5ff3 \ + --hash=sha256:a88370a7d89fcb1c4953a87673fdd7b4a0eb14a1a4dfce49771f0c827ef44893 \ + --hash=sha256:ab6db7d2a18d366cc57c2228253cf26443190aba0a6dd0939b3c1e8ac6e29e2c \ + --hash=sha256:ad81bf81b0a0b6c6ec74169638202851962843e86749570c463eecc55072f93b \ + --hash=sha256:aed178902c2386d7c5d3d23eb96d32c100e34cb8c2390e7ece0e4901ae43f0e7 \ + --hash=sha256:b0c82c19baca8ddeb4f513f584f53f6d3aa96b1a273f1a507d6d70620b01ba92 \ + --hash=sha256:b238e955ba34ef2b8897f358b7b868b41b9a02ffd338014b62985fa91898cc4a \ + --hash=sha256:b40f814df9e106371fea48911814383284e99df34ec1aa1fdd9b07d2055345d0 \ + --hash=sha256:b40fb47d637df8da7b02d76f242688416c23e53195ea5748895db671c01759d2 \ + --hash=sha256:bc5c0203d383403043fb86c964bd0bab4fcbfb26004ff4bb9c6d02ebc1d608ae \ + --hash=sha256:bde5d1b37101b1e9dd3da1f35072e2e7028e9c5e3511f7d76d3fdd4d071b7663 \ + --hash=sha256:bfaa998ceeea4d0aa72b40cdd0023d19409504e244b439ff2aa9f01729341c5f \ + --hash=sha256:c25c594f58ecb676358d6d6b0ff068b8bbbc506dc831c6d17876460c66ce39c2 \ + --hash=sha256:c39c7130ea0702c4ab0faf12da1df1e02d5174305c17edf02309e2f058c4114f \ + --hash=sha256:c40f3b1cd3ff9dd9f4ae829e4301f0d3a553e3467058b8c3f5528fee2c768a20 \ + --hash=sha256:c44dd9881626da7d621c23805f26726f6b023cf3e9755f48d092bc9cbef4a8e7 \ + --hash=sha256:c4d9c76e9a16a8bae0bdcc57efabad499192565bd9a95258b01fb0b49a62bd63 \ + --hash=sha256:c6e6c226b1ca5402d7ae5fb34a0d21f1b49124fe4200e5884d1e19e53c47ac1d \ + --hash=sha256:ca7b967e96384abdf7e7182c79f71529997981ece8169f8a8ddb31bc5b57cbec \ + --hash=sha256:cab37b82ec328173222e4f9da5eec4f2ec9e8e506f83557c8be8e1bffad351cc \ + --hash=sha256:ce3889e3815f97d46414eb574bffdd9bdb41ff70f503097e2707615a87d4e92c \ + --hash=sha256:cef2a8f006410b6134a0d273ec037fea8cc7a6a914f1bd7555ad9788ad788c6e \ + --hash=sha256:cf63fffcdcd8c60f223d3967bb92cc4fc2e8b46f09e75b67a6a75e6f47c0fc43 \ + --hash=sha256:d5b665a43fe0d3b390cbdd3c003d61c92fa07bd5e3fb1ed3f47920c2d03cd9fd \ + --hash=sha256:d6d274ec50a5b208be75596dc44ea253e65deaa6ee3a600babc86dafbb957dfc \ + --hash=sha256:d800c7689154622b0ba2922ceca44a3cf2ef61c3b9a4c4eeb1d8b3050d7ededa \ + --hash=sha256:d90c91cb4ef83b2ff00db4e0a7bdd9602902504ef9b26d0f9d7ecf6cd05c7554 \ + --hash=sha256:da42395e7add724c1f7caf18a2977b1fbdfd5aab314e5622731f0ed66731eaaf \ + --hash=sha256:da847332447db5505162759a4cd5ac374eb8b74841fe97a98ef3de14edd2586d \ + --hash=sha256:dc401274fcc7b15b3b2c12df2ff34024a11925243a7d3daee91c6d7d14f9addf \ + --hash=sha256:df6e3a36170cda0d313be50fe5065948e7f12f3a181b38cbc262e9f2ee4824e1 \ + --hash=sha256:e089a22ff5af1290b8c759a610830bdb2a829ef9c3d7797e4ee32c2f795ed482 \ + --hash=sha256:e85a9db9e5a5ccc326edb19e35a5106ba16e451d570a2ec8ea9deb1ea52a3c42 \ + --hash=sha256:ea27bcf5c56b13463ba5b9bbfa4d6544997e47ba6db77c59a259b09daa802d4d \ + --hash=sha256:f063c696328408fc4f259b9d7d439398d36b709e12445a904e7b047f0a84c3c5 \ + --hash=sha256:f1630201b0e2a96bb26304b7adfbd91a4ef486abb5a4c48377444a0bed749f37 \ + --hash=sha256:f1c911818fb076910ef509f2298dfcb966a54a6ff068eebd459632102cf589fb \ + --hash=sha256:f280c115ea64eff3dcbd68a668ce3f63476a4ba386bbabb318017e286196ea2c \ + --hash=sha256:f595bb0185aab3e9dc31950c95d914f56ea8278810c3b928f3426e12ed6d27bc \ + --hash=sha256:f98eaf784cd12bc69c77af398084174531007cd81849c962163ccfc6e791f3ea \ + --hash=sha256:fc0eb73b450b53950b7879ac7642889c82918d17bd2d877fd7270348dfd5550c \ + --hash=sha256:fcccaa1484f7dd1091602970988ab741491f9f974013c844f70e45ac1196b80d \ + --hash=sha256:fd3f878a4aac3c262447ddf43c5f4c18fc67dfc3ba69c4fb1c7a4c4af96abe7e + # via aiobotocore +yarl==1.25.1 \ + --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ + --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ + --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ + --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ + --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ + --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ + --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ + --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ + --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ + --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ + --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ + --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ + --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ + --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ + --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ + --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ + --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ + --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ + --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ + --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ + --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ + --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ + --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ + --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ + --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ + --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ + --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ + --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ + --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ + --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ + --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ + --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ + --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ + --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ + --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ + --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ + --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ + --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ + --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ + --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ + --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ + --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ + --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ + --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ + --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ + --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ + --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ + --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ + --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ + --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ + --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ + --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ + --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ + --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ + --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ + --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ + --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ + --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ + --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ + --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ + --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ + --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ + --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ + --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ + --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ + --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ + --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ + --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ + --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ + --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ + --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ + --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ + --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ + --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ + --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ + --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ + --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ + --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ + --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ + --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ + --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ + --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ + --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ + --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ + --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ + --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ + --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ + --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ + --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ + --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ + --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ + --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ + --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ + --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ + --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ + --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ + --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ + --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ + --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ + --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ + --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ + --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ + --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ + --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ + --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ + --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ + --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ + --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ + --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ + --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ + --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ + --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ + --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ + --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ + --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ + --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ + --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ + --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ + --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ + --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ + --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ + --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ + --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ + --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ + --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ + --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ + --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ + --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ + --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ + --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ + --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ + --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ + --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ + --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ + --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ + --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ + --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 + # via aiohttp diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py new file mode 100644 index 00000000..8a68c548 --- /dev/null +++ b/scripts/collect_evidence.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Run every check of REPORT.md on the current checkout and save the real output in evidence/. + + python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet + +Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the +stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the +lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records +container states before/after every hook. Prints no secrets +(hook output is shown by the public tests as suppressed; our scripts print names and states only). +On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin. +""" +from __future__ import annotations + +import datetime as dt +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +WINDOWS = os.name == "nt" +HOOK = ".cmd" if WINDOWS else ".sh" +PYTHON = sys.executable +VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python")) +# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path. +MAKE_PY = "python" if WINDOWS else "python3" +MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python" +ENV = {**os.environ, "PYTHONUTF8": "1"} +if WINDOWS: + ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin" + + +def now() -> str: + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]: + started = time.monotonic() + result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + timeout=timeout, env={**ENV, **(env or {})}) + took = time.monotonic() - started + return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n" + + +class Log: + def __init__(self, name: str, title: str): + self.path = EVIDENCE / name + self.path.parent.mkdir(parents=True, exist_ok=True) + commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True, + encoding="utf-8").stdout.strip() + self.text = f"# {title}\n# {now()}, commit {commit}\n" + + def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int: + code, out = sh(cmd, **kwargs) + self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}" + self.save() + print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True) + return code + + def note(self, text: str) -> None: + self.text += f"\n{text}\n" + self.save() + + def save(self) -> None: + self.path.write_text(self.text, encoding="utf-8") + + +def states() -> list[str]: + ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split() + rows = [] + for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"): + st = item["State"] + rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} " + f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}") + return sorted(rows) + + +def main() -> int: + EVIDENCE.mkdir(exist_ok=True) + make = shutil.which("make", path=ENV["PATH"]) or "make" + # 1. Clean clone -> secrets -> build -> start + log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions") + log.run(["git", "status", "--short", "--branch"]) + log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"]) + log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version") + log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py") + if not (ROOT / ".env").exists(): + shutil.copy(ROOT / ".env.example", ROOT / ".env") + log.note("$ cp .env.example .env") + log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"]) + log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800) + log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"]) + log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"]) + log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts + log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"]) + # venv for validate / own tests + if not Path(VENV_PY).exists(): + log.run([PYTHON, "-m", "venv", ".venv"]) + log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800) + log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800) + + # 2. make check / validate / test + log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test") + log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check") + log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}") + log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test") + + # 3. persistence with states before/after + log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3") + log.note("Containers BEFORE:\n" + "\n".join(states())) + log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"], + shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}") + log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states())) + log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) + + # 4. failover: until BOTH replicas have been the stopped one (each run is a real public test) + log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat") + stopped: set[str] = set() + import urllib.request + for attempt in range(1, 7): + # The public test stops the replica that served its first POST; its request sequence is + # fixed, so with round robin it hits the same replica every time. Shift the rotation by + # (attempt - 1) extra requests so that the other replica becomes the target as well. + for _ in range(attempt - 1): + urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read() + before = {r for r in states() if "-api-" in r and " running " in r} + log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before))) + log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"], + shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}") + after = states() + gone = [r for r in after if "-api-" in r and " exited " in r] + log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone)) + log.run([PYTHON, "scripts/lbctl.py", "status"]) + stopped.update(r.split()[0] for r in gone) + log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"], + shown="sh scripts/restore-api.sh") + log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) + if len(stopped) >= 2: + break + log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}") + + # 5. own tests + log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally") + log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v", + timeout=3600) + + # 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script + log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py") + log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600) + log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800) + log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600) + log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) + log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"]) + log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"]) + print("done:", EVIDENCE) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/demo_attachments.py b/scripts/demo_attachments.py new file mode 100644 index 00000000..21f98c21 --- /dev/null +++ b/scripts/demo_attachments.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids, +download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated +request straight to S3 (from inside the storage network) is refused, the signed one works. + + python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080] + +Prints requests and answers; never prints tokens or keys. Creates synthetic users. +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import subprocess +import sys +import urllib.error +import urllib.request +import uuid +from email.message import Message +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "tests" / "fixtures" / "sample.png" +PASSWORD = "Demo-password-Ω-12345" + + +class Client: + def __init__(self, base: str): + self.base = base.rstrip("/") + + def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None, + headers: dict | None = None) -> tuple[int, Message, bytes]: + h = dict(headers or {}) + if token: + h["Authorization"] = "Bearer " + token + data = raw + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode() + h["Content-Type"] = "application/json" + req = urllib.request.Request(self.base + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, r.read() # case-insensitive HTTPMessage + except urllib.error.HTTPError as e: + return e.code, e.headers, e.read() + + +def show(label: str, status: int, headers: Message, payload: bytes, *, binary: bool = False) -> None: + inst = headers.get("X-Instance-Id", "-") + if binary: + text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}" + else: + text = payload.decode("utf-8", "replace")[:300] + print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}") + + +def user(c: Client, label: str) -> dict: + name = f"{label}_{uuid.uuid4().hex[:8]}" + status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(), + "password": PASSWORD}) + assert status == 201, body + basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode() + status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + session = json.loads(body) + return {"id": session["user"]["id"], "token": session["token"], "name": name} + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://127.0.0.1:8080") + args = p.parse_args() + c = Client(args.base_url) + content = FIXTURE.read_bytes() + print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}") + alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve") + print(f"users: alice, bob (members), eve (outsider)") + status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"], + body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]}) + chat = json.loads(body) + + print("\n1. upload -> message -> download by the second member") + st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"], + raw=content, headers={"Content-Type": "image/png"}) + show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b) + att = json.loads(b) + st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"], + body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b) + st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"]) + show("GET /attachments/{id} (bob)", st, h, b) + st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"]) + show("GET /attachments/{id}/content (bob)", st, h, got, binary=True) + same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"] + print(f" SHA-256 of the download == fixture == metadata: {same}") + + print("\n2. outsider and direct S3 access") + for suffix in ("", "/content"): + st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"]) + show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b) + key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"], + cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + print(f" object key in PostgreSQL: {key} (random; not in any API response: " + f"{key not in json.dumps(att)})") + probe = ( + "import urllib.request as u, urllib.error as e, asyncio\n" + "def st(url):\n" + " try: return u.urlopen(url, timeout=3).status\n" + " except e.HTTPError as x: return x.code\n" + f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n" + "print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n" + "from messenger.config import Settings\nfrom messenger.storage import Storage\n" + "async def m():\n s = Storage(Settings.from_env()); await s.open()\n" + f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n" + "asyncio.run(m())\n") + out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT, + capture_output=True, text=True).stdout + for line in out.splitlines(): + print(" (inside network storage) " + line) + print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).") + return 0 if same else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 00000000..0a929edb --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4). + + python scripts/explain_history.py --output evidence/explain-history.txt + +Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in +30 other chats, then explains the EXACT statements of src/messenger/repo.py: + 1. first page (HISTORY_FIRST) + 2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement + reused by the driver would get it + 3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan + 4. the rejected OFFSET pagination for the same late page +Runs psql inside the postgres container; no database port is published. +""" +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL) + +CHAT = "00000000-0000-4000-8000-00000000e002" +USER = "00000000-0000-4000-8000-00000000e001" +TARGET = 20_000 +LATE = 19_950 # cursor position: the 50 newest messages remain + +SEED = f""" +\\set ON_ERROR_STOP on +BEGIN; +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING; +INSERT INTO chats (id, title, created_by, created_at) +SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'), + '{USER}', timestamptz '2026-01-01 00:00:00+00' +FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats +ON CONFLICT DO NOTHING; +INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) +SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'), + timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM generate_series(1::int, {TARGET}::int) AS g +WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}'); +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'), + timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g +WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}' + AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id); +COMMIT; +ANALYZE messages; +""" + + +def pg(sql_param: str) -> str: + """Turn psycopg %s placeholders into $1..$n for a PREPARE statement.""" + parts = sql_param.split("%s") + return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts)) + + +EXPLAIN = f""" +\\set ON_ERROR_STOP on +\\pset pager off +SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages; +SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}' +ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset +SET plan_cache_mode = force_generic_plan; +PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)}; +PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)}; +PREPARE or_page(uuid, timestamptz, uuid, int) AS + SELECT id, sender_id, text, created_at FROM messages + WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4; +PREPARE offset_page(uuid, int, int) AS + SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3; +\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51); +\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51); +\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51); +\\echo '=== 4. REJECTED: OFFSET pagination, same late page' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51); +""" + + +def psql(script: str) -> str: + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"], + cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True) + return result.stdout + + +def run() -> str: + psql(SEED) + return psql(EXPLAIN) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--output", default=None) + args = parser.parse_args() + text = run() + print(text) + if args.output: + Path(args.output).write_text(text, encoding="utf-8", newline="\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..04e5d62b --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Create local synthetic credentials for the stand in ./secrets (ignored by Git). + + python scripts/init_secrets.py + +Idempotent: existing files are kept, so re-running never changes passwords of a +database that already has data. Prints only file names, never values. +Delete ./secrets together with the volumes (`docker compose down -v`) to start over. +""" +from __future__ import annotations + +import os +import secrets +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + path = SECRETS / name + if path.exists(): + print(f"keep secrets/{name}") + return + path.write_text(value + "\n", encoding="utf-8", newline="\n") + # Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the + # compose bind mount. Local synthetic lab credentials only. + os.chmod(path, 0o644) + print(f"created secrets/{name}") + + +def read(name: str) -> str: + return (SECRETS / name).read_text(encoding="utf-8").strip() + + +def main() -> int: + SECRETS.mkdir(exist_ok=True) + write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs + write("valkey_password", secrets.token_urlsafe(24)) + write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger") + write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0") + # Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): + # admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List. + for who in ("admin", "api", "reconciler"): + write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper()) + write(f"s3_{who}_secret_access_key", secrets.token_hex(20)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/lbctl.py b/scripts/lbctl.py new file mode 100644 index 00000000..259d35ec --- /dev/null +++ b/scripts/lbctl.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""What the load balancer thinks right now, mapped to real containers. + + python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id + python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy + python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state + +Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`, +so no admin port is published. Container hostname = X-Instance-Id of that replica. +Prints no secrets. Used by student-tests and bench scripts as a library as well. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping. +OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"} +ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it) + + +def run(*args: str, timeout: float = 60) -> str: + result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8", + timeout=timeout) + if result.returncode != 0: + raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}") + return result.stdout + + +def replicas(all_states: bool = True) -> list[dict]: + """API containers of this compose project (running and stopped).""" + ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split() + if not ids: + return [] + data = json.loads(run("docker", "inspect", *ids)) + result = [] + for item in data: + networks = item["NetworkSettings"]["Networks"] or {} + result.append({ + "id": item["Id"][:12], + "name": item["Name"].lstrip("/"), + "hostname": item["Config"]["Hostname"], + "state": item["State"]["Status"], + "health": (item["State"].get("Health") or {}).get("Status", "-"), + "exit_code": item["State"]["ExitCode"], + "started_at": item["State"]["StartedAt"], + "pid": item["State"]["Pid"], + "ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")), + # compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...} + "networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")}, + }) + return sorted(result, key=lambda r: r["name"]) + + +def lb_servers() -> list[dict]: + raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c", + 'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock') + by_ip = {ip: r for r in replicas() for ip in r["ips"]} + servers = [] + for line in raw.splitlines(): + parts = line.split() + if len(parts) < 7 or not parts[0].isdigit(): + continue + name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6]) + if admin & ADMIN_RMAINT or addr == "-": + state = "MAINT" + else: + state = OP_STATE.get(op, str(op)) + container = by_ip.get(addr) + servers.append({"slot": name, "addr": addr, "state": state, + "container": container["name"] if container else None, + "instance_id": container["hostname"] if container else None}) + return servers + + +def up_instances() -> set[str]: + return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"} + + +def wait_up(count: int, timeout: float = 60) -> float: + """Seconds until exactly `count` servers are UP (raises TimeoutError otherwise).""" + started = time.monotonic() + while True: + try: + if len(up_instances()) == count: + return time.monotonic() - started + except RuntimeError: + pass # lb restarting + if time.monotonic() - started > timeout: + raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s") + time.sleep(0.25) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + sub.add_parser("replicas") + wait = sub.add_parser("wait-up") + wait.add_argument("count", type=int) + wait.add_argument("--timeout", type=float, default=60) + args = parser.parse_args() + if args.command == "status": + for s in lb_servers(): + if s["state"] != "MAINT" or s["container"]: + print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}") + print(f"(other slots: MAINT, no container)") + elif args.command == "replicas": + for r in replicas(): + print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} " + f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}") + else: + took = wait_up(args.count, args.timeout) + print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..f8a4ff9a --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,6 @@ +@echo off +rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows +rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash. +rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh" +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..f00dcb4f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. +# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS). +# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed. +# Output is suppressed by the test; run it by hand to see the steps. +set -eu +cd "$(dirname "$0")/.." + +# 1. Entry point and API first: no request is served while the stores restart underneath. +docker compose stop lb api +# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer +# flush), then start again. The S3 bytes live on the named volume s3data. +docker compose restart postgres valkey s3 +# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves +# the new replica addresses through Docker DNS by itself - no stale backend list. +docker compose start api +docker compose start lb +echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved" diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 00000000..78a06845 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker +# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s); +# `python scripts/lbctl.py wait-up 2` waits for that explicitly. +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-recreate --no-deps --wait api +docker compose ps api --format '{{.Name}} {{.Status}}' diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..5a8e4263 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Rolling update of the API without downtime (Compose has no built-in one for a scaled service). +# 1. build the new image from the working tree; +# 2. run migrations ONCE with the new image (they must be additive / compatible with the +# running old version: expand first, contract in a later release); +# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until +# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully +# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it. +# At every moment at least N replicas serve traffic. Stops on the first error (set -e); +# the old replicas that were not replaced yet keep running. +set -eu +cd "$(dirname "$0")/.." +PY="${PYTHON:-$(command -v python3 || command -v python)}" +# New image = new revision label (git commit + UTC time), visible in docker inspect and logs. +BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}" +export BUILD_REVISION + +replicas="$(docker compose ps -q api | wc -l | tr -d ' ')" +[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; } +old="$(docker compose ps -q api)" +echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION" + +docker compose build api +docker compose run --rm --no-deps migrate + +for id in $old; do + name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')" + docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api + "$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60 + docker stop --time 15 "$id" >/dev/null + docker rm "$id" >/dev/null + "$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60 + echo "rolling-update: replaced $name" +done +for id in $(docker compose ps -q api); do + docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id" +done diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..1292249d --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows +rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash; +rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment. +rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..85bc37ad --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`. +# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname, +# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are +# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy); +# restore it explicitly afterwards: scripts/restore-api.sh. +# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s) +# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process +# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed). +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac +mode="${STOP_MODE:-stop}" + +target="" +# Only RUNNING containers of THIS compose project's service `api` are candidates. +for id in $(docker compose ps -q api); do + if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then + target="$id" + fi +done +if [ -z "$target" ]; then + echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2 + exit 3 +fi +name="$(docker inspect -f '{{.Name}}' "$target")" +case "$mode" in + stop) docker stop --time 15 "$target" >/dev/null ;; + kill) docker stop --time 0 "$target" >/dev/null ;; + *) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;; +esac +echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")" diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..26148a23 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,3 @@ +"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»).""" + +__version__ = "1.0.0" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..ef22eaf7 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,88 @@ +"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker). + +Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy): +1. readiness flips to 503 (`app.state.draining = True`), requests are still served; +2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has + marked this replica DOWN and sends nothing new here; +3. uvicorn closes the listening socket; in-flight requests get up to + SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish; +4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0. + DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL. +A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey. +""" +from __future__ import annotations + +import logging +import os +import sys +import threading +from types import FrameType + +import uvicorn + +from . import __version__, logging_setup +from .app import create_app +from .config import ConfigError, Settings + +log = logging.getLogger("messenger") + + +class GracefulServer(uvicorn.Server): + """uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop.""" + + def __init__(self, config: uvicorn.Config, drain_seconds: float): + super().__init__(config) + self.drain_seconds = drain_seconds + self.drain_timer: threading.Timer | None = None + + def handle_exit(self, sig: int, frame: FrameType | None) -> None: + self.config.app.state.draining = True # config.app is our Starlette instance + if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit: + log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds, + "timeout_s": self.config.timeout_graceful_shutdown}}) + # uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe. + self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame)) + self.drain_timer.daemon = True + self.drain_timer.start() + return + if self.drain_timer is not None: + self.drain_timer.cancel() # second signal: stop draining now + self._stop(sig, frame) + + def _stop(self, sig: int, frame: FrameType | None) -> None: + if not self.should_exit: + log.info("shutdown_requested", extra={"fields": {"signal": sig}}) + super().handle_exit(sig, frame) + + +def main() -> int: + try: + settings = Settings.from_env() + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + logging_setup.configure(settings.log_level) + app = create_app(settings) + config = uvicorn.Config( + app, + host=settings.host, + port=settings.port, + log_config=None, + access_log=False, + server_header=False, + proxy_headers=False, + lifespan="on", + timeout_graceful_shutdown=settings.shutdown_timeout, + ) + server = GracefulServer(config, settings.drain_seconds) + log.info("starting", extra={"fields": { + "version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port, + "python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id, + }}) + server.run() + log.info("stopped") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..38100ad2 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,584 @@ +"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping.""" +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import json +import logging +import random +import re +import time +import uuid +from typing import Any + +from starlette.applications import Starlette +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import JSONResponse, Response, StreamingResponse +from starlette.routing import Route +from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send + +from . import attachments, authhdr, cursor, docs, passwords, repo, validation +from .config import Settings +from .db import Database +from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable +from .sessions import SessionStore, build_store +from .storage import ObjectMissing, Storage +from .timefmt import rfc3339 + +log = logging.getLogger("messenger") +access_log = logging.getLogger("messenger.access") + +REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +# -------------------------------------------------------------------------- +# Responses +# -------------------------------------------------------------------------- +def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse: + body = {"error": {"code": code, "message": message, "request_id": request_id}} + return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})}) + + +async def api_error_handler(request: Request, exc: Exception) -> JSONResponse: + assert isinstance(exc, ApiError) + return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers) + + +async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse: + """Unknown routes and wrong methods also answer in the contract error format.""" + assert isinstance(exc, HTTPException) + if exc.status_code == 404: + return error_response(request.state.request_id, 404, "not_found", "Объект не найден") + if exc.status_code == 405: + return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers) + return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос") + + +# -------------------------------------------------------------------------- +# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions) +# -------------------------------------------------------------------------- +class RequestContextMiddleware: + """Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503 + included), writes one access-log line, turns crashes into 500.""" + + def __init__(self, app: ASGIApp, instance_id: str): + self.app = app + self.instance_id = instance_id.encode("ascii") + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1") + request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex + scope.setdefault("state", {})["request_id"] = request_id + status = 500 + started = False + begin = time.perf_counter() + + async def send_with_id(message: AsgiMessage) -> None: + nonlocal status, started + if message["type"] == "http.response.start": + started = True + status = message["status"] + headers = [(k, v) for k, v in message.get("headers", []) + if k.lower() not in (b"x-request-id", b"x-instance-id")] + headers.append((b"x-request-id", request_id.encode("latin-1"))) + headers.append((b"x-instance-id", self.instance_id)) + message = {**message, "headers": headers} + await send(message) + + try: + await self.app(scope, receive, send_with_id) + except asyncio.CancelledError: + # uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires: + # answer 503 (retryable) instead of a generic 500, then let the cancellation proceed. + if not started: + status = 503 + response = error_response(request_id, 503, "unavailable", "Сервер завершает работу") + await response(scope, receive, send_with_id) + raise + except Exception: + log.exception("unhandled_error", extra={"fields": {"request_id": request_id}}) + if not started: + status = 500 + response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера") + await response(scope, receive, send_with_id) + finally: + access_log.info("request", extra={"fields": { + "request_id": request_id, + "method": scope["method"], + "path": scope["path"], # ids only; query (cursor) and bodies are never logged + "status": status, + "instance_id": self.instance_id.decode("ascii"), + "duration_ms": round((time.perf_counter() - begin) * 1000, 3), + }}) + + +# -------------------------------------------------------------------------- +# Request helpers +# -------------------------------------------------------------------------- +def db_of(request: Request) -> Database: + return request.app.state.db + + +def sessions_of(request: Request) -> SessionStore: + return request.app.state.sessions + + +def storage_of(request: Request) -> Storage: + return request.app.state.storage + + +async def authenticate(request: Request) -> str: + """User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2. + + Store failures are 503 (Unavailable), never 401 and never anonymous access. + """ + token = authhdr.parse_bearer(request.headers.get("authorization")) + user_id = await sessions_of(request).user_id_for(token) + if user_id is None: + raise Unauthorized("Сессия недействительна, истекла или отозвана") + return user_id + + +def path_uuid(request: Request, name: str) -> str: + return validation.parse_uuid(request.path_params[name], name) + + +async def read_json(request: Request) -> dict[str, Any]: + content_type = request.headers.get("content-type", "") + media_type, _, params = content_type.partition(";") + if media_type.strip().lower() != "application/json": + raise InvalidRequest("Content-Type должен быть application/json") + charset = params.strip().lower() + if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}: + raise InvalidRequest("Поддерживается только кодировка UTF-8") + limit: int = request.app.state.settings.max_body_bytes + declared = request.headers.get("content-length") + if declared is not None and declared.isdigit() and int(declared) > limit: + raise PayloadTooLarge() + chunks: list[bytes] = [] + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise PayloadTooLarge() + chunks.append(chunk) + return validation.parse_json_object(b"".join(chunks)) + + +def page_params(request: Request) -> tuple[int, str | None]: + query = request.query_params.multi_items() + limit = validation.parse_limit(validation.single_query_param(query, "limit")) + raw_cursor = validation.single_query_param(query, "cursor") + if raw_cursor is not None and raw_cursor == "": + raise InvalidRequest("cursor не может быть пустым") + return limit, raw_cursor + + +async def reject_body(request: Request) -> None: + """POST /auth/sessions takes credentials only from the Basic header; any body is 400.""" + if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0": + raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела") + async for chunk in request.stream(): + if chunk: + raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела") + + +def idempotency_key(request: Request) -> str: + values = request.headers.getlist("idempotency-key") + return validation.idempotency_key(values[0] if len(values) == 1 else None) + + +def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes: + """SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter.""" + canonical = json.dumps({"attachment_ids": attachment_ids, "text": text}, + ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +def ok(body: Any, status: int = 200) -> JSONResponse: + return JSONResponse(body, status_code=status) + + +# -------------------------------------------------------------------------- +# Handlers +# -------------------------------------------------------------------------- +async def live(request: Request) -> JSONResponse: + return ok({"status": "ok"}) + + +async def ready(request: Request) -> JSONResponse: + """200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey + (SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments) + answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first.""" + if request.app.state.draining: + raise Unavailable("Сервер завершает работу") + db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(), + storage_of(request).ping()) + if not db_ok: + raise Unavailable("PostgreSQL недоступен") + if not store_ok: + raise Unavailable("Хранилище сессий недоступно") + if not s3_ok: + raise Unavailable("Хранилище файлов (S3) недоступно") + return ok({"status": "ok"}) + + +async def ready_core(request: Request) -> JSONResponse: + """Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only. + If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to + replicas that are at least core-ready, so chats and messages keep working while every + attachment request answers 503 itself (see infra/haproxy/haproxy.cfg).""" + if request.app.state.draining: + raise Unavailable("Сервер завершает работу") + db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping()) + if not (db_ok and store_ok): + raise Unavailable("PostgreSQL или хранилище сессий недоступно") + return ok({"status": "ok"}) + + +async def register_user(request: Request) -> JSONResponse: + data = await read_json(request) + validation.check_fields(data, {"username", "display_name", "password"}) + username = validation.username_field(data) + display_name = validation.text_field(data, "display_name", 100) + password = validation.password_field(data) + password_hash = await passwords.hash_password(password) + user = await repo.create_user(db_of(request), username, display_name, password_hash) + log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}}) + return ok(user.to_json(), 201) + + +async def create_session(request: Request) -> JSONResponse: + """Login: only `Authorization: Basic`; wrong user and wrong password give the same 401.""" + username, password = authhdr.parse_basic(request.headers.get("authorization")) + await reject_body(request) + found = await repo.get_credentials(db_of(request), username) + user, stored_hash = found if found else (None, None) + if not await passwords.verify_password(stored_hash, password) or user is None: + log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}}) + raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE) + ttl = request.app.state.settings.session_ttl_seconds + token, expires_at = await sessions_of(request).create(user.id, ttl) + log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}}) + return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201) + + +async def revoke_session(request: Request) -> Response: + token = authhdr.parse_bearer(request.headers.get("authorization")) + if not await sessions_of(request).revoke(token): + raise Unauthorized("Сессия недействительна, истекла или отозвана") + log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}}) + return Response(status_code=204) + + +async def me(request: Request) -> JSONResponse: + user_id = await authenticate(request) + user = await repo.get_user(db_of(request), user_id) + if user is None: + raise Unauthorized("Сессия недействительна, истекла или отозвана") + return ok(user.to_json()) + + +async def create_chat(request: Request) -> JSONResponse: + user_id = await authenticate(request) + data = await read_json(request) + validation.check_fields(data, {"title", "member_ids"}) + title = validation.text_field(data, "title", 100) + member_ids = validation.member_ids_field(data) + chat = await repo.create_chat(db_of(request), user_id, title, member_ids) + log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}}) + return ok(chat.to_json(), 201) + + +async def list_chats(request: Request) -> JSONResponse: + user_id = await authenticate(request) + limit, raw_cursor = page_params(request) + after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None + items, has_more = await repo.list_chats(db_of(request), user_id, after, limit) + next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None + return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor}) + + +async def get_chat(request: Request) -> JSONResponse: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json()) + + +def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]: + """`text` is required and a string (also with attachments); `attachment_ids`, if present, an + array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments.""" + validation.check_fields(data, {"text"}, frozenset({"attachment_ids"})) + attachment_ids = validation.attachment_ids_field(data) + text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1, + allow_blank=bool(attachment_ids)) + return text, attachment_ids + + +async def create_message(request: Request) -> JSONResponse: + """Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body + -> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session.""" + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + db = db_of(request) + settings = request.app.state.settings + if settings.message_write_path == "multi": + await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at + key = idempotency_key(request) + text, attachment_ids = parse_message(await read_json(request)) + result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text, + settings.idempotency_ttl_seconds, attachment_ids) + else: + try: + key = idempotency_key(request) + text, attachment_ids = parse_message(await read_json(request)) + except ApiError: + # Rare error path only: an outsider must still get 404, not a hint about the body. + await repo.ensure_member(db, user_id, chat_id) + raise + # Membership check, key claim, chat clock and message: one SQL call, one transaction. + result = await repo.post_message_single_call(db, user_id, chat_id, key, + request_fingerprint(text, attachment_ids), text, + settings.idempotency_ttl_seconds, attachment_ids) + if result.replayed: + log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id, + "message_id": result.message.id}}) + return ok(result.message.to_json(), 201) + + +async def list_messages(request: Request) -> JSONResponse: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + db = db_of(request) + await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak + limit, raw_cursor = page_params(request) + after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None + items, has_more = await repo.list_messages(db, chat_id, after, limit) + next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None + return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor}) + + +# ------------------------------------------------------------------ attachments (lab 4) +def fault(request: Request, point: str) -> None: + """Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only).""" + if point in request.app.state.settings.fault_points: + log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}}) + raise Unavailable("Внедрённый отказ: " + point) + + +async def upload_attachment(request: Request) -> JSONResponse: + """POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py).""" + settings: Settings = request.app.state.settings + reader = attachments.BodyReader(request) + slots: asyncio.Semaphore = request.app.state.upload_slots + acquired = False + try: + try: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + db = db_of(request) + await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body + filename = validation.filename_param(validation.query_items(request.scope["query_string"])) + declared = attachments.declared_type(request) + length = attachments.declared_length(request, settings.max_upload_bytes) + try: + await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds) + except asyncio.TimeoutError: + raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None + acquired = True + upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length) + except ApiError: + # Early rejection: drop (not store) a bounded rest of the body so the client reads our error. + await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0) + raise + # The slot is held until the buffer is no longer needed (after the S3 PUT), so at most + # UPLOAD_CONCURRENCY bodies of this replica exist in memory at any moment. + return await store_upload(request, db, chat_id, user_id, filename, upload) + finally: + if acquired: + slots.release() + + +async def store_upload(request: Request, db: Database, chat_id: str, user_id: str, filename: str, + upload: attachments.Upload) -> JSONResponse: + """intent (pending) -> S3 PUT -> ready -> 201 (order and failure analysis: docs/CONSISTENCY.md).""" + attachment_id = uuid.uuid4() + object_key = attachments.new_object_key() + storage = storage_of(request) + await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data), + upload.sha256, object_key) + await storage.put(object_key, upload.data, upload.info.content_type, + {"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()}) + fault(request, "after_s3_put") + attachment = await attachments.mark_ready(db, attachment_id) + if attachment is None: + raise Unavailable("Загрузка не завершена, повторите запрос") + fault(request, "after_ready_commit") + log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id, "chat_id": chat_id, + "size": attachment.size_bytes, "type": attachment.content_type}}) + return ok(attachment.to_json(), 201) + + +async def get_attachment(request: Request) -> JSONResponse: + user_id = await authenticate(request) + attachment_id = path_uuid(request, "attachment_id") + return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json()) + + +def content_disposition(filename: str) -> str: + """RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted.""" + from urllib.parse import quote + fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename) + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}" + + +async def download_attachment(request: Request) -> Response: + """Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from + S3 in 64 KiB chunks (memory per download does not depend on the file size).""" + user_id = await authenticate(request) + attachment_id = path_uuid(request, "attachment_id") + query = validation.query_items(request.scope["query_string"]) + try: + validation.variant_param(query) + except validation.NotFoundVariant: + await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first + raise NotFound("Такого варианта у вложения нет") from None + attachment = await attachments.get_visible(db_of(request), user_id, attachment_id) + if attachment.lost: + raise NotFound("Оригинал вложения утрачен") + try: + size, chunks = await storage_of(request).open_stream(attachment.object_key) + except ObjectMissing: + # A ready row without its object: data loss, reported by the reconciler (quarantine). + log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id}}) + raise NotFound("Оригинал вложения утрачен") from None + if size != attachment.size_bytes: + log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id}}) + await chunks.aclose() + raise Unavailable("Оригинал вложения временно недоступен") + headers = { + "Content-Length": str(size), + "Content-Disposition": content_disposition(attachment.filename), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + attachment.sha256 + '"', + } + return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers) + + +def create_app(settings: Settings | None = None) -> Starlette: + settings = settings or Settings.from_env() + db = Database(settings) + sessions = build_store(settings, db) + storage = Storage(settings) + + async def purge_loop() -> None: + """Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS).""" + while True: + await asyncio.sleep(600) + try: + deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds) + if deleted: + log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}}) + except ApiError: + pass # database unavailable right now; next round + + async def janitor_loop(every: float) -> None: + """Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py). + Every replica runs it; the advisory lock lets one round run at a time.""" + while True: + await asyncio.sleep(every * (0.5 + random.random())) + try: + removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds) + if removed: + log.info("stale_uploads_removed", extra={"fields": {"count": removed}}) + except ApiError: + pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round + + stats_log = logging.getLogger("messenger.stats") + + async def stats_loop(every: float) -> None: + """Experiments only: DB pool saturation (waiting requests, wait time) and process CPU.""" + cpu = time.process_time() + db.pool.pop_stats() + while True: + await asyncio.sleep(every) + stats = db.pool.pop_stats() + now_cpu = time.process_time() + stats_log.info("pool_stats", extra={"fields": { + "interval_s": every, "cpu_s": round(now_cpu - cpu, 4), + **{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num", + "requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}}) + cpu = now_cpu + + @contextlib.asynccontextmanager + async def lifespan(app: Starlette): + passwords.configure(settings.hash_concurrency) + await db.open() + await storage.open() + log.info("dependencies_configured", extra={"fields": { + "session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds, + "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds, + "instance_id": settings.instance_id, "drain_s": settings.drain_seconds, + "message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket, + "upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency, + "pending_grace_s": settings.pending_grace_seconds, + "fault_points": sorted(settings.fault_points)}}) + tasks = [asyncio.create_task(purge_loop())] + if settings.janitor_interval_seconds > 0: + tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds))) + if settings.stats_log_seconds > 0: + tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds))) + try: + yield + finally: + for task in tasks: + task.cancel() + await sessions.close() + await storage.close() + await db.close() + + routes = [ + Route("/health/live", live, methods=["GET"]), + Route("/health/ready", ready, methods=["GET"]), + Route("/health/ready/core", ready_core, methods=["GET"]), + Route("/api/v1/users", register_user, methods=["POST"]), + Route("/api/v1/users/me", me, methods=["GET"]), + Route("/api/v1/auth/sessions", create_session, methods=["POST"]), + Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]), + Route("/api/v1/chats", create_chat, methods=["POST"]), + Route("/api/v1/chats", list_chats, methods=["GET"]), + Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]), + Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]), + Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]), + ] + if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract + routes += [ + Route("/docs", docs.swagger_page, methods=["GET"]), + Route("/openapi.json", docs.openapi_json, methods=["GET"]), + ] + app = Starlette( + routes=routes, + exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler}, + lifespan=lifespan, + ) + app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id) + app.state.settings = settings + app.state.db = db + app.state.sessions = sessions + app.state.storage = storage + app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency) + app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency) + app.state.draining = False + return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 00000000..ec670a60 --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,300 @@ +"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket. + +Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order: + + auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413) + -> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After) + -> body in chunks: size limit, read deadline, format + pixel count from the first bytes + -> full integrity check (images.verify) -> SHA-256 / size known + -> INSERT row status='pending' with a fresh random object_key, COMMIT (intent) + -> PUT object to S3 (Content-MD5) + -> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready. + +Why 201 is honest: it is sent only after BOTH the object and the ready row are durable. +What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md): + +* before the intent COMMIT: nothing (no object was written); +* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the + object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the + object - idempotent, a missing key is fine - and then the row, once it is older than + PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of + COMMIT we do not know whether the row is ready, and deleting the object could destroy a + ready attachment. Only the janitor decides, under a row lock, from the committed state; +* after the ready COMMIT (answer lost): a correct ready attachment the client does not know. +""" +from __future__ import annotations + +import asyncio +import datetime as dt +import hashlib +import logging +import secrets +import uuid +from dataclasses import dataclass + +import psycopg +from psycopg import errors as pg_errors +from starlette.requests import Request + +from . import images +from .config import Settings +from .db import Database +from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType +from .storage import OBJECT_PREFIX, Storage +from .timefmt import rfc3339 + +log = logging.getLogger("messenger.attachments") + +JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas + + +@dataclass(frozen=True, slots=True) +class Attachment: + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + object_key: str + status: str + created_at: dt.datetime + updated_at: dt.datetime + lost: bool = False + + def to_json(self) -> dict: + """Public representation: no object key, no bucket, no internal state.""" + return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename, + "content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256, + "status": "ready", "variants": [], "error_code": None, + "created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)} + + +COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, " + "a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL") + + +def _attachment(row) -> Attachment: + return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7], + row[8], row[9], row[10], row[11]) + + +def new_object_key() -> str: + """Server-generated, 128 random bits, unrelated to the attachment id and the filename.""" + return OBJECT_PREFIX + secrets.token_hex(16) + + +# ------------------------------------------------------------------ SQL +async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str, + info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None: + async def work(conn: psycopg.AsyncConnection) -> None: + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')", + (attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height, + object_key)) + try: + await db.run(work, transaction=False) + except pg_errors.ForeignKeyViolation: + raise NotFound("Чат не найден") from None + + +async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None: + """pending -> ready. None if the row is no longer pending (the janitor removed a stale intent).""" + async def work(conn: psycopg.AsyncConnection) -> Attachment | None: + cur = await conn.execute( + "UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() " + f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,)) + row = await cur.fetchone() + return _attachment(row) if row else None + return await db.run(work, transaction=False) + + +async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment: + """A READY attachment of a chat the user is a member of; anything else is 404 (unknown, + foreign chat, unfinished upload) - existence of foreign objects is never revealed.""" + async def work(conn: psycopg.AsyncConnection) -> Attachment: + cur = await conn.execute( + f"SELECT {COLUMNS} FROM attachments a " + "JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s " + "WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id)) + row = await cur.fetchone() + if row is None: + raise NotFound("Вложение не найдено") + return _attachment(row) + return await db.run(work, transaction=False) + + +async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int: + """Janitor round: remove intents of uploads that never became ready. + + Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in + mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object. + Order: object first (idempotent), row second; a crash in between leaves the pending row and + the next round repeats. Only status='pending' rows older than the grace period are touched: + fresh uploads and ready/used attachments are never selected. + """ + async def work(conn: psycopg.AsyncConnection) -> int: + cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,)) + if not (await cur.fetchone())[0]: + return 0 + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'pending' " + "AND created_at < clock_timestamp() - make_interval(secs => %s) " + "ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch)) + rows = await cur.fetchall() + if not rows: + return 0 + await asyncio.gather(*(storage.delete(key) for _, key in rows)) + await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'", + ([r[0] for r in rows],)) + return len(rows) + return await db.run(work, deadline=storage.timeout + 5) + + +# ------------------------------------------------------------------ request body +class Busy(Unavailable): + """All upload slots of this replica are taken: retry shortly (no body was read).""" + + @property + def headers(self) -> dict[str, str]: + return {"Retry-After": "1"} + + +@dataclass +class Upload: + data: bytearray + info: images.ImageInfo + sha256: bytes + + +class BodyReader: + """Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it + is consumed and can discard the rest after an early rejection.""" + + def __init__(self, request: Request): + self.receive = request.receive + self.done = False + + async def chunk(self, timeout: float) -> bytes | None: + if self.done: + return None + message = await asyncio.wait_for(self.receive(), timeout) + if message["type"] == "http.disconnect": + self.done = True + raise InvalidRequest("Клиент прервал загрузку") + if not message.get("more_body", False): + self.done = True + return message.get("body", b"") + + async def discard(self, limit: int, timeout: float) -> None: + """After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that + writes the whole body before reading the answer still gets our JSON error instead of a + connection reset. Nothing is kept in memory; beyond the limit the connection is dropped.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + seen = 0 + try: + while not self.done and seen <= limit: + remaining = deadline - loop.time() + if remaining <= 0: + return + seen += len(await self.chunk(remaining) or b"") + except (ApiError, asyncio.TimeoutError): + return + + +def declared_type(request: Request) -> str: + """Content-Type of the upload must be image/png or image/jpeg (parameters ignored).""" + media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower() + if media_type not in (images.PNG, images.JPEG): + raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg") + return media_type + + +def declared_length(request: Request, limit: int) -> int | None: + raw = request.headers.get("content-length") + if raw is None: + return None # chunked transfer: the limit is enforced while reading + if not raw.isdigit(): + raise InvalidRequest("Неверный Content-Length") + length = int(raw) + if length > limit: + raise PayloadTooLarge(f"Файл больше {limit} байт") + if length == 0: + raise InvalidRequest("Тело запроса пустое: нужны байты изображения") + return length + + +async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore, + length: int | None) -> Upload: + """Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly + Content-Length bytes allocated once when the length is declared (no growth, no final copy); + it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known + after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without + reading/decoding the rest.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + settings.upload_read_timeout_seconds + limit = settings.max_upload_bytes + buffer = bytearray(length) if length else bytearray() + view = memoryview(buffer) if length else None + filled = 0 + digest = hashlib.sha256() + info: images.ImageInfo | None = None + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise InvalidRequest("Тело запроса не получено вовремя") + try: + chunk = await reader.chunk(remaining) + except asyncio.TimeoutError: + raise InvalidRequest("Тело запроса не получено вовремя") from None + if chunk is None: + break + if filled + len(chunk) > (length or limit): + raise PayloadTooLarge(f"Файл больше {limit} байт") + if view is not None: + view[filled:filled + len(chunk)] = chunk + else: + buffer += chunk + filled += len(chunk) + digest.update(chunk) + if info is None and filled: + info = _sniff(view[:filled] if view is not None else buffer, settings, declared) + if view is not None: + view.release() + if not filled: + raise InvalidRequest("Тело запроса пустое: нужны байты изображения") + if length and filled != length: + raise InvalidRequest("Тело запроса короче Content-Length") + if info is None: + raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG") + try: + async with verify_slots: + await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds, + jpeg_max_memory_mb=settings.jpeg_max_memory_mb) + except images.ImageError as exc: + raise _image_error(exc) from None + except asyncio.TimeoutError: + raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None + except (images.VerifierUnavailable, MemoryError): + log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}}) + raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None + return Upload(buffer, info, digest.digest()) + + +def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None: + try: + info = images.sniff(buffer, settings.max_image_pixels) + except images.ImageError as exc: + raise _image_error(exc) from None + if info is not None and info.content_type != declared: + # "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa). + raise UnsupportedMediaType("Содержимое не совпадает с Content-Type") + return info + + +def _image_error(exc: images.ImageError) -> ApiError: + return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message) + diff --git a/src/messenger/authhdr.py b/src/messenger/authhdr.py new file mode 100644 index 00000000..f849b029 --- /dev/null +++ b/src/messenger/authhdr.py @@ -0,0 +1,48 @@ +"""Parsing of the Authorization header. Any malformed value is 401, never 500. + +Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs +Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the +local synthetic lab stand (lab 7 adds TLS). +""" +from __future__ import annotations + +import base64 +import binascii +import re + +from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized +from .validation import USERNAME_RE + +B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}") +TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters + + +def _split(header: str | None) -> tuple[str, str]: + if not header: + return "", "" + scheme, _, value = header.strip().partition(" ") + return scheme.lower(), value.strip() + + +def parse_basic(header: str | None) -> tuple[str, str]: + """(username, password) from `Basic base64(username:password)`; the password may contain ':'.""" + invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE) + scheme, value = _split(header) + if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024: + raise invalid + try: + decoded = base64.b64decode(value, validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise invalid from None + username, sep, password = decoded.partition(":") # split at the FIRST colon + if not sep or USERNAME_RE.fullmatch(username) is None or not password: + raise invalid + return username, password + + +def parse_bearer(header: str | None) -> str: + scheme, value = _split(header) + if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None: + # Missing header, Basic on a business endpoint, garbage token: one answer. + raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE) + return value diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..1772442a --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,236 @@ +"""Runtime configuration read from environment variables (12-factor style). + +Non-secret settings have safe defaults. Connection strings with passwords are +read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in +/run/secrets), so they never appear in `docker inspect`, compose files or logs. +Invalid values stop the process at startup with a clear message. +""" +from __future__ import annotations + +import os +import re +import socket +from dataclasses import dataclass +from pathlib import Path + + +class ConfigError(ValueError): + pass + + +INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}") + + +def _instance_id() -> str: + """X-Instance-Id of this process: INSTANCE_ID or the container hostname. + + Docker sets the hostname of a container to its short container ID, so the value + is unique per replica, stable for the life of the process and maps 1:1 to a real + container (`docker inspect`), which scripts/stop-one.sh relies on. + """ + value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + if INSTANCE_ID_RE.fullmatch(value) is None: + raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}") + return value + + +def _int(name: str, default: int, low: int, high: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + try: + value = int(raw.strip()) + except ValueError: + raise ConfigError(f"{name} must be an integer, got {raw!r}") from None + if not low <= value <= high: + raise ConfigError(f"{name} must be in [{low}, {high}], got {value}") + return value + + +def _float(name: str, default: float, low: float, high: float) -> float: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + try: + value = float(raw.strip()) + except ValueError: + raise ConfigError(f"{name} must be a number, got {raw!r}") from None + if not low <= value <= high: + raise ConfigError(f"{name} must be in [{low}, {high}], got {value}") + return value + + +def _bool(name: str, default: bool) -> bool: + raw = os.environ.get(name, "").strip().lower() + if raw == "": + return default + if raw not in {"true", "false", "1", "0"}: + raise ConfigError(f"{name} must be true/false, got {raw!r}") + return raw in {"true", "1"} + + +def _choice(name: str, default: str, allowed: set[str]) -> str: + value = os.environ.get(name, "").strip().lower() or default + if value not in allowed: + raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}") + return value + + +def _secret(name: str, required: bool) -> str: + """Value of _FILE (preferred, Docker secret) or (local runs/tests).""" + path = os.environ.get(f"{name}_FILE", "").strip() + if path: + try: + value = Path(path).read_text(encoding="utf-8").strip() + except OSError as exc: + # The message names the file, never its content. + raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None + else: + value = os.environ.get(name, "").strip() + if required and not value: + raise ConfigError(f"{name}_FILE (or {name}) must be set") + return value + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + env: str + log_level: str + shutdown_timeout: float + max_body_bytes: int + docs_enabled: bool = True + database_url: str = "" + session_store: str = "postgres" + cache_url: str = "" + session_ttl_seconds: int = 3600 + db_pool_min: int = 1 + db_pool_max: int = 10 + db_timeout_seconds: float = 3.0 + cache_timeout_seconds: float = 0.5 + hash_concurrency: int = 2 + instance_id: str = "local" + drain_seconds: float = 0.0 + idempotency_ttl_seconds: int = 86400 + stats_log_seconds: float = 0.0 + message_write_path: str = "function" + # Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py). + s3_endpoint: str = "http://s3:8333" + s3_region: str = "us-east-1" + s3_bucket: str = "messenger-attachments" + s3_access_key_id: str = "" + s3_secret_access_key: str = "" + s3_timeout_seconds: float = 3.0 + s3_connect_timeout_seconds: float = 1.0 + s3_max_connections: int = 20 + max_upload_bytes: int = 10 * 1024 * 1024 + max_image_pixels: int = 40_000_000 + upload_concurrency: int = 4 + upload_queue_timeout_seconds: float = 5.0 + upload_read_timeout_seconds: float = 30.0 + image_verify_timeout_seconds: float = 10.0 + image_verify_concurrency: int = 1 + jpeg_max_memory_mb: int = 256 + pending_grace_seconds: int = 900 + janitor_interval_seconds: float = 60.0 + fault_points: frozenset[str] = frozenset() + + @property + def db_statement_timeout_ms(self) -> int: + # The server stops a statement a bit before the client-side deadline fires. + return max(100, int(self.db_timeout_seconds * 1000) - 500) + + @classmethod + def from_env(cls) -> "Settings": + log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info" + if log_level not in {"debug", "info", "warning", "error"}: + raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}") + session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres" + if session_store == "redis": + session_store = "valkey" # same protocol; the course allows Redis OR Valkey + if session_store not in {"postgres", "valkey"}: + raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}") + db_pool_min = _int("DB_POOL_MIN", 1, 0, 50) + db_pool_max = _int("DB_POOL_MAX", 10, 1, 100) + if db_pool_min > db_pool_max: + raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX") + return cls( + host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0", + port=_int("APP_PORT", 8080, 1, 65535), + env=os.environ.get("APP_ENV", "lab").strip() or "lab", + log_level=log_level, + # DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period. + shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0), + max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024), + docs_enabled=_bool("DOCS_ENABLED", True), + database_url=_secret("DATABASE_URL", required=True), + session_store=session_store, + cache_url=_secret("CACHE_URL", required=session_store == "valkey"), + # Contract name. Minimum 1 s so the expiry test can use 2 s. + session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600), + db_pool_min=db_pool_min, + db_pool_max=db_pool_max, + db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0), + cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0), + hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16), + instance_id=_instance_id(), + # After SIGTERM: readiness is 503 but requests are still served for this long, + # so the load balancer stops routing here before the listener closes. + drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0), + # Contract: a replay within 24 h returns the original result. + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600), + # Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off. + stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0), + # Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous + # multi-statement transaction; both have identical semantics (A/B on one image). + message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}), + **s3_settings(), + ) + + +# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them. +CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 +# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs. +FAULT_POINTS = {"after_s3_put", "after_ready_commit"} + + +def s3_settings(*, credentials_required: bool = True) -> dict: + """S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys.""" + faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip()) + if faults - FAULT_POINTS: + raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}") + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333" + if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint): + raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments" + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket): + raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}") + return dict( + s3_endpoint=endpoint, + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=bucket, + s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required), + s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required), + s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0), + s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0), + s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200), + max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES), + max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS), + # At most this many request bodies of one replica are buffered/checked at once (RAM bound: + # UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503. + upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64), + upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0), + upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0), + image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0), + # Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive + # file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB). + image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16), + jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048), + # An intent row still `pending` after this long belongs to a failed upload: the janitor + # deletes its object and the row. Far above the longest possible upload (~45 s). + pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600), + janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0), + fault_points=faults, + ) diff --git a/src/messenger/cursor.py b/src/messenger/cursor.py new file mode 100644 index 00000000..8e2d5ee6 --- /dev/null +++ b/src/messenger/cursor.py @@ -0,0 +1,56 @@ +"""Opaque keyset cursors. + +A cursor encodes the sort key `(created_at, id)` of the last item on the page +plus the resource it belongs to, e.g. `v1|m|||`, +packed in URL-safe base64. The client treats it as an opaque string; the server +rejects cursors of another chat/user (400) and any non-canonical encoding. +""" +from __future__ import annotations + +import base64 +import re + +from .errors import InvalidRequest +from .validation import UUID_RE + +B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}") +TS_RE = re.compile(r"[0-9]{1,19}") + +# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z. +MAX_TS_US = 253402300799999999 + +MESSAGES = "m" +CHATS = "c" + +SortKey = tuple[int, str] + + +def encode(scope: str, owner_id: str, key: SortKey) -> str: + ts, item_id = key + raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii") + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def decode(cursor: str, scope: str, owner_id: str) -> SortKey: + """Return the sort key stored in the cursor or raise InvalidRequest.""" + bad = InvalidRequest("Неверный cursor") + if B64_RE.fullmatch(cursor) is None: + raise bad + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii") + except (ValueError, UnicodeDecodeError): + raise bad from None + parts = raw.split("|") + if len(parts) != 5 or parts[0] != "v1": + raise bad + _, got_scope, got_owner, ts, item_id = parts + if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower(): + raise bad + key = (int(ts), item_id) + if key[0] > MAX_TS_US: + raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL + if encode(got_scope, got_owner, key) != cursor: + raise bad # only the canonical form produced by the server is accepted + if got_scope != scope or got_owner != owner_id: + raise InvalidRequest("cursor относится к другому ресурсу") + return key diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..1c81bfdd --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,131 @@ +"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work. + +Three independent limits keep a request from hanging when the database misbehaves: + +* pool `timeout` — waiting for a free connection (pool exhausted); +* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow + statement or a statement stuck behind a lock; +* client deadline — covers what the server cannot report, e.g. a + paused/frozen PostgreSQL container or a dead network. + +The client deadline needs care. Every unit of work runs as its own task; at the deadline +the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a +psycopg query makes psycopg send a cancel request, and libpq starts that connection +(PQcancelStart) synchronously in the event-loop thread, resolving the host name first. +When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network +disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute. +The abandoned task instead ends by itself, bounded by: + +* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network + breaks the socket within seconds, the connection is discarded by the pool; +* `statement_timeout` / `lock_timeout` on a live but slow server; +* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits + at most the pool timeout and gets 503. + +Any failure becomes 503 `unavailable` — never 401, never a hang. +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import TypeVar + +import psycopg +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from .config import Settings +from .errors import Unavailable + +log = logging.getLogger("messenger.db") + +T = TypeVar("T") +Work = Callable[[psycopg.AsyncConnection], Awaitable[T]] + +# Errors that mean "the database is unavailable right now", not "the request is wrong". +# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown +# (server restart) are subclasses of OperationalError. +UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError) + + +def _forget(task: asyncio.Task) -> None: + """Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'.""" + if not task.cancelled(): + task.exception() + + +class Database: + def __init__(self, settings: Settings): + self.deadline = settings.db_timeout_seconds + options = ( + f"-c statement_timeout={settings.db_statement_timeout_ms} " + f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} " + "-c idle_in_transaction_session_timeout=10000" + ) + self.pool = AsyncConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_timeout_seconds, # max wait for a free connection + open=False, + name="messenger", + check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart + kwargs={ + "options": options, + "connect_timeout": max(1, int(settings.db_timeout_seconds)), + "application_name": "messenger-api", + # Autocommit connections: a single statement (post_message function, ping) is its + # own transaction with ONE round trip; multi-statement work uses an explicit + # conn.transaction() (BEGIN ... COMMIT), see run(). + "autocommit": True, + # Detect a dead peer / lost network without any cancel request (see module doc). + "keepalives": 1, + "keepalives_idle": 5, + "keepalives_interval": 1, + "keepalives_count": 3, + "tcp_user_timeout": int(settings.db_timeout_seconds * 1000), + }, + ) + + async def open(self) -> None: + # wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet; + # readiness stays 503 until the database answers. + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close(timeout=2) + + async def run(self, work: Work[T], deadline: float | None = None, *, transaction: bool = True) -> T: + """Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception). + transaction=False: `work` issues exactly ONE statement, which autocommit makes atomic. + + Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged; + database unavailability and the deadline become Unavailable (503). + """ + async def unit() -> T: + async with self.pool.connection() as conn: + if not transaction: + return await work(conn) + async with conn.transaction(): + return await work(conn) + + task = asyncio.ensure_future(unit()) + try: + return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline) + except UNAVAILABLE_ERRORS as exc: + log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}}) + raise Unavailable("База данных временно недоступна") from None + finally: + if not task.done(): + # Deadline or cancelled request: stop waiting now. The task is left to finish + # on its own (no task.cancel(): that would trigger psycopg's blocking cancel). + task.add_done_callback(_forget) + + async def ping(self, timeout: float = 1.0) -> bool: + async def select_one(conn: psycopg.AsyncConnection) -> None: + await conn.execute("SELECT 1") + try: + await self.run(select_one, deadline=timeout, transaction=False) + return True + except Unavailable: + return False diff --git a/src/messenger/docs.py b/src/messenger/docs.py new file mode 100644 index 00000000..cb2ea46e --- /dev/null +++ b/src/messenger/docs.py @@ -0,0 +1,57 @@ +"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs + +The page shows the course contract contracts/openapi.json as is (it is not +generated from our code). Only `servers` is replaced with "/" so that the +"Try it out" requests go to the same host the page was opened from. +Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN. +Disable with DOCS_ENABLED=false. +""" +from __future__ import annotations + +import json +from functools import lru_cache +from pathlib import Path + +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse + +SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json" +SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0" + +PAGE = f""" + + + + Messenger lab1 — Swagger UI + + + +
+ + + + +""" + + +@lru_cache(maxsize=1) +def _spec() -> dict: + spec = json.loads(SPEC_PATH.read_text(encoding="utf-8")) + spec["servers"] = [{"url": "/", "description": "этот сервер"}] + return spec + + +async def swagger_page(request: Request) -> HTMLResponse: + return HTMLResponse(PAGE) + + +async def openapi_json(request: Request) -> JSONResponse: + return JSONResponse(_spec()) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..fa3ce51a --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,71 @@ +"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md).""" +from __future__ import annotations + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +class ApiError(Exception): + status = 500 + code = "internal_error" + default_message = "Внутренняя ошибка сервера" + + def __init__(self, message: str | None = None): + self.message = message or self.default_message + super().__init__(self.message) + + @property + def headers(self) -> dict[str, str]: + return {} + + +class InvalidRequest(ApiError): + status = 400 + code = "invalid_request" + default_message = "Неверный запрос" + + +class Unauthorized(ApiError): + """401 with the challenge of the scheme the endpoint expects (RFC 7235).""" + + status = 401 + code = "unauthorized" + default_message = "Требуется действительная сессия" + + def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE): + super().__init__(message) + self.challenge = challenge + + @property + def headers(self) -> dict[str, str]: + return {"WWW-Authenticate": self.challenge} + + +class NotFound(ApiError): + status = 404 + code = "not_found" + default_message = "Объект не найден" + + +class Conflict(ApiError): + status = 409 + code = "conflict" + default_message = "Конфликт" + + +class PayloadTooLarge(ApiError): + status = 413 + code = "payload_too_large" + default_message = "Тело запроса слишком большое" + + +class UnsupportedMediaType(ApiError): + status = 415 + code = "unsupported_media_type" + default_message = "Неподдерживаемый тип содержимого" + + +class Unavailable(ApiError): + status = 503 + code = "unavailable" + default_message = "Сервис временно недоступен" diff --git a/src/messenger/healthcheck.py b/src/messenger/healthcheck.py new file mode 100644 index 00000000..8411514b --- /dev/null +++ b/src/messenger/healthcheck.py @@ -0,0 +1,19 @@ +"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`.""" +from __future__ import annotations + +import os +import sys +import urllib.request + + +def main() -> int: + port = os.environ.get("APP_PORT", "8080").strip() or "8080" + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response: + return 0 if response.status == 200 else 1 + except Exception: + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/images.py b/src/messenger/images.py new file mode 100644 index 00000000..7169c0c2 --- /dev/null +++ b/src/messenger/images.py @@ -0,0 +1,328 @@ +"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says. + +Two stages, both with bounded memory and both BEFORE anything is stored: + +1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises + PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn). + A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels + (413) before the rest of the body is read and long before any pixel is decoded. +2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread: + * PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is + inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly + height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND + must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail. + Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB). + * JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale + (DCT scaling: the output raster is 64 times smaller) in a child process; any decoder + warning is fatal, so a truncated file or a file without scan data after SOS fails. + +Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413); +nothing here may turn user input into a 500. +""" +from __future__ import annotations + +import asyncio +import os +import shutil +import struct +import zlib +from dataclasses import dataclass + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SOI = b"\xff\xd8\xff" + +PNG = "image/png" +JPEG = "image/jpeg" + +# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it. +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +# Adam7 passes: (x start, y start, x step, y step). +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC). +JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} +# Standalone markers without a length field. +JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)} + +DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile +INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once +INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call +MAX_DIMENSION = 2**31 - 1 + + +class ImageError(Exception): + """The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413.""" + + def __init__(self, status: int, message: str): + super().__init__(message) + self.status = status + self.message = message + + +class VerifierUnavailable(Exception): + """The checker itself could not run to the end (resources): the API answers 503, retryable.""" + + +def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError: + return ImageError(415, message) + + +@dataclass(frozen=True, slots=True) +class ImageInfo: + content_type: str + width: int + height: int + + +def detect(head: bytes) -> str | None: + """Format by signature; None while fewer bytes than a signature have arrived.""" + if head[:8] == PNG_SIGNATURE: + return PNG + if head[:3] == JPEG_SOI: + return JPEG + probe = min(len(head), 8) + if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]): + return None + raise unsupported() + + +def sniff(head: bytes, max_pixels: int) -> ImageInfo | None: + """Format and size from the beginning of the file, or None if more bytes are needed. + + Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels.""" + kind = detect(head) + if kind is None: + return None + info = _png_header(head) if kind == PNG else _jpeg_header(head) + if info is None: + return None + if info.width * info.height > max_pixels: + raise ImageError(413, f"Изображение больше {max_pixels} пикселей") + return info + + +def _png_header(head: bytes) -> ImageInfo | None: + if len(head) < 33: + return None + length, ctype = struct.unpack(">I4s", head[8:16]) + if ctype != b"IHDR" or length != 13: + raise unsupported() + if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]: + raise unsupported() + width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29]) + if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION): + raise unsupported() + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1: + raise unsupported() + return ImageInfo(PNG, width, height) + + +def _jpeg_header(head: bytes) -> ImageInfo | None: + """Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet.""" + i = 2 + n = len(head) + while True: + if i >= n: + return None + if head[i] != 0xFF: + raise unsupported() + while i < n and head[i] == 0xFF: # fill bytes + i += 1 + if i >= n: + return None + marker = head[i] + i += 1 + if marker in JPEG_STANDALONE: + continue + if marker in (0xD8, 0xD9, 0xDA, 0x00): + # Second SOI, EOI or scan data before any frame header: not a usable JPEG. + raise unsupported() + if i + 2 > n: + return None + length = struct.unpack(">H", head[i:i + 2])[0] + if length < 2: + raise unsupported() + if marker in JPEG_SOF: + if i + 7 > n: + return None + if length < 8: + raise unsupported() + height, width = struct.unpack(">HH", head[i + 3:i + 7]) + if width == 0 or height == 0: # height 0 = DNL-defined height: not supported + raise unsupported() + return ImageInfo(JPEG, width, height) + i += length + + +# ------------------------------------------------------------------ full verification +async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0, + jpeg_max_memory_mb: int = 256) -> None: + """Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib + releases the GIL); JPEG: a strict libjpeg decoder in a child process.""" + if info.content_type == PNG: + await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout) + else: + await verify_jpeg(data, timeout, jpeg_max_memory_mb) + + +def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]: + """(byte count, row stride) of every pass of the raw (filtered) image data.""" + bpp = PNG_CHANNELS[color] * depth + passes = ADAM7 if interlace else ((0, 0, 1, 1),) + segments = [] + for xs, ys, dx, dy in passes: + pw = (width - xs + dx - 1) // dx if width > xs else 0 + ph = (height - ys + dy - 1) // dy if height > ys else 0 + if pw and ph: + stride = 1 + (pw * bpp + 7) // 8 + segments.append((ph * stride, stride)) + return segments + + +class _RawChecker: + """Counts inflated bytes against the expected size and checks the filter byte of every row + using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast.""" + + VALID_FILTERS = bytes(range(5)) + + def __init__(self, segments: list[tuple[int, int]]): + self.segments = segments + self.index = 0 # current pass + self.offset = 0 # bytes already seen inside the current pass + self.expected = sum(size for size, _ in segments) + self.total = 0 + + def feed(self, piece: bytes) -> None: + self.total += len(piece) + if self.total > self.expected: + raise unsupported() + pos = 0 + while pos < len(piece): + size, stride = self.segments[self.index] + take = min(size - self.offset, len(piece) - pos) + part = piece[pos:pos + take] + first = (-self.offset) % stride + if part[first::stride].translate(None, self.VALID_FILTERS): + raise unsupported() + self.offset += take + pos += take + if self.offset == size: + self.index += 1 + self.offset = 0 + + +def verify_png(data: bytes | bytearray | memoryview) -> None: + view = memoryview(data) + if bytes(view[:8]) != PNG_SIGNATURE: + raise unsupported() + pos = 8 + n = len(view) + checker: _RawChecker | None = None + inflater = zlib.decompressobj() + color = None + seen_plte = seen_idat = idat_done = False + first = True + while True: + if pos + 12 > n: + raise unsupported() # truncated: no room for another chunk header + CRC + length, ctype = struct.unpack(">I4s", view[pos:pos + 8]) + if length > n - pos - 12: + raise unsupported() + body = view[pos + 8:pos + 8 + length] + crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0] + if zlib.crc32(body, zlib.crc32(ctype)) != crc: + raise unsupported() + pos += 12 + length + if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype): + raise unsupported() + if first: + if ctype != b"IHDR" or length != 13: + raise unsupported() + width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body) + if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS + or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1): + raise unsupported() + checker = _RawChecker(_png_segments(width, height, depth, color, interlace)) + first = False + continue + assert checker is not None + if ctype == b"IHDR": + raise unsupported() + if ctype == b"PLTE": + if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768: + raise unsupported() + seen_plte = True + elif ctype == b"IDAT": + if idat_done: + raise unsupported() # IDAT chunks must be consecutive + if color == 3 and not seen_plte: + raise unsupported() + seen_idat = True + _inflate(inflater, body, checker) + elif ctype == b"IEND": + if length != 0 or not seen_idat: + raise unsupported() + if pos != n: + raise unsupported() # bytes after IEND (polyglot files) are not accepted + break + elif not 97 <= ctype[0] <= 122: + raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image + if ctype != b"IDAT" and seen_idat: + idat_done = True + # The zlib stream must end exactly with the last pixel row, without trailing data. + if not inflater.eof or inflater.unused_data or checker.total != checker.expected: + raise unsupported() + + +def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None: + if inflater.eof: + if len(body): + raise unsupported() # data after the end of the zlib stream + return + # Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so + # feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step. + try: + for start in range(0, len(body), INPUT_STEP): + pending: bytes | memoryview = body[start:start + INPUT_STEP] + while pending: + if inflater.eof: + raise unsupported() # data after the end of the zlib stream + checker.feed(inflater.decompress(pending, INFLATE_STEP)) + pending = inflater.unconsumed_tail + if inflater.eof and inflater.unused_data: + raise unsupported() + except zlib.error: + raise unsupported() from None + + +async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None: + """Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode. + + `-strict` turns every libjpeg warning into a failure: "premature end of data segment" + (scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data. + Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for + a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x + smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient + buffers of progressive files); the separate process is killed at the deadline. + """ + process = await asyncio.create_subprocess_exec( + DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull, + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + stderr = b"" + try: + _, stderr = await asyncio.wait_for(process.communicate(data), timeout) + except (asyncio.TimeoutError, asyncio.CancelledError): + process.kill() + await process.wait() + raise + except (BrokenPipeError, ConnectionResetError): + await process.wait() # djpeg stopped reading: the file is already known to be broken + if process.returncode is not None and process.returncode < 0: + # Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the + # SERVER, not of the file - never report it as 415. + raise VerifierUnavailable() + if process.returncode != 0: + if b"Insufficient memory" in (stderr or b""): + # A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption. + raise ImageError(413, "Изображение слишком велико для обработки") + raise unsupported() diff --git a/src/messenger/logging_setup.py b/src/messenger/logging_setup.py new file mode 100644 index 00000000..388f14de --- /dev/null +++ b/src/messenger/logging_setup.py @@ -0,0 +1,36 @@ +"""JSON-lines logging to stdout. Logs never contain message texts or request bodies.""" +from __future__ import annotations + +import datetime as dt +import json +import logging +import sys + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + entry = { + "ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"), + "level": record.levelname.lower(), + "logger": record.name, + "event": record.getMessage(), + } + fields = getattr(record, "fields", None) + if fields: + entry.update(fields) + if record.exc_info: + # Stack traces go only to the server log, never to the HTTP client. + entry["exception"] = self.formatException(record.exc_info) + return json.dumps(entry, ensure_ascii=False) + + +def configure(level: str) -> None: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + root.handlers[:] = [handler] + root.setLevel(level.upper()) + # uvicorn's own access log is disabled; our middleware writes one line per request. + for name in ("uvicorn", "uvicorn.error"): + logging.getLogger(name).handlers[:] = [] + logging.getLogger(name).propagate = True diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..10b18e9f --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,76 @@ +"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`). + +* Files `migrations/NNNN_name.sql` are applied in name order, each in its own + transaction together with its row in `schema_migrations` (DDL is transactional + in PostgreSQL, so a failed migration leaves nothing behind). +* Re-running is safe: applied versions are skipped, so data is never duplicated. +* The SHA-256 of every applied file is stored; if an applied file was edited later, + the run stops instead of silently diverging from the real schema. +* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once). +""" +from __future__ import annotations + +import hashlib +import sys +import time +from pathlib import Path + +import psycopg + +from .config import ConfigError, _secret + +MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations" +LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id + + +def say(text: str) -> None: + print(f"migrate: {text}", flush=True) + + +def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection: + deadline = time.monotonic() + wait_seconds + while True: + try: + return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate") + except psycopg.OperationalError as exc: + if time.monotonic() > deadline: + raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None + time.sleep(1) + + +def main() -> int: + try: + url = _secret("DATABASE_URL", required=True) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + with connect(url) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + try: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY, checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT now())") + applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()) + for path in files: + version, sql = path.stem, path.read_bytes() + checksum = hashlib.sha256(sql).hexdigest() + if version in applied: + if applied[version] != checksum: + say(f"{version}: file changed after it was applied; create a new migration instead") + return 1 + say(f"{version}: already applied, skipped") + continue + with conn.transaction(): + conn.execute(sql.decode("utf-8")) + conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum)) + say(f"{version}: applied") + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + say(f"done, {len(files)} migration file(s) checked") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/passwords.py b/src/messenger/passwords.py new file mode 100644 index 00000000..a1fc35ad --- /dev/null +++ b/src/messenger/passwords.py @@ -0,0 +1,53 @@ +"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7). + +* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt + there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole. +* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker + thread (argon2-cffi releases the GIL), so the event loop keeps serving other + requests, and a semaphore caps parallel hashes (memory/CPU DoS protection). +* An unknown username is checked against a dummy hash, so "no such user" and + "wrong password" take the same time and return the same 401. +""" +from __future__ import annotations + +import asyncio + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16) +_semaphore: asyncio.Semaphore | None = None +_dummy_hash: str | None = None + + +def configure(concurrency: int) -> None: + global _semaphore + _semaphore = asyncio.Semaphore(concurrency) + + +async def _run(func, *args): + if _semaphore is None: + configure(2) + async with _semaphore: + return await asyncio.to_thread(func, *args) + + +async def hash_password(password: str) -> str: + return await _run(_hasher.hash, password) + + +def _verify(stored_hash: str, password: str) -> bool: + try: + return _hasher.verify(stored_hash, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +async def verify_password(stored_hash: str | None, password: str) -> bool: + global _dummy_hash + if stored_hash is None: + if _dummy_hash is None: + _dummy_hash = await hash_password("dummy-password-for-timing") + await _run(_verify, _dummy_hash, password) + return False + return await _run(_verify, stored_hash, password) diff --git a/src/messenger/reconcile.py b/src/messenger/reconcile.py new file mode 100644 index 00000000..e2457b3b --- /dev/null +++ b/src/messenger/reconcile.py @@ -0,0 +1,437 @@ +"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL. + + docker compose run --rm reconciler # dry-run: report only (default) + docker compose run --rm reconciler --apply # fix what is safe to fix + docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256) + +How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2 +pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key` +(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size; +cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md). + +What it finds and what --apply does (nothing else is ever changed): + + kind condition --apply + orphan_object object, no row, older than --grace delete the object + stale_intent row 'pending' older than --grace (+/- object) delete object, then row + missing_object row 'ready', object absent, older than --grace quarantine the row + size_mismatch row 'ready', object size != size_bytes quarantine the row + checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row + restored_object row quarantined, object present and correct again remove the quarantine + (a checksum quarantine only with --verify-checksums) + fresh_* the same, but younger than --grace nothing (in-flight upload) + +Safety: +* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with + default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in + the planner cannot modify a store; +* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined, + which is reversible; objects that have a row are never deleted unless the row is a stale + intent; anything younger than --grace is never touched; +* every action re-checks its condition right before acting, under a row lock (FOR UPDATE) + where a row exists, so the live API racing with the reconciler cannot be hurt; +* object first, row second: an interrupted run leaves a state the next run repairs; + re-running is idempotent (the second run finds nothing to do); +* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned + objects (pointing it at the wrong/empty database would otherwise wipe the bucket). +""" +from __future__ import annotations + +import argparse +import asyncio +import datetime as dt +import hashlib +import json +import sys +import time +from collections import Counter +from collections.abc import AsyncIterator +from dataclasses import dataclass, field + +import psycopg + +from .config import ConfigError, Settings, _float, _secret, s3_settings +from .errors import Unavailable +from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage + +KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch", + "restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object") +MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3) +ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine", + "size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"} + + +def log(text: str) -> None: + print(f"reconcile: {text}", file=sys.stderr, flush=True) + + +@dataclass(frozen=True, slots=True) +class Row: + id: str + key: str + status: str + size: int + sha256: bytes + created_at: dt.datetime + updated_at: dt.datetime + lost_reason: str | None + used: bool + + +@dataclass(slots=True) +class Finding: + kind: str + key: str + attachment_id: str | None = None + used: bool = False + detail: str = "" + result: str = "planned" # planned | applied | skipped: | failed: + + +@dataclass +class Stats: + objects: int = 0 + rows: int = 0 + consistent: int = 0 + s3_list_requests: int = 0 + s3_head_requests: int = 0 + s3_get_bytes: int = 0 + db_pages: int = 0 + + +class ReadOnlyStorage: + """The dry-run view of S3: listing and reading only - there is no delete to call.""" + + def __init__(self, storage: Storage): + self._s = storage + + async def list_page(self, *args, **kwargs): + return await self._s.list_page(*args, **kwargs) + + async def head(self, key: str): + return await self._s.head(key) + + async def read_all(self, key: str, limit: int) -> bytes: + return await self._s.read_all(key, limit) + + +class Reconciler: + def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float, + page_size: int, verify: bool, apply: bool): + self.conn = conn + self.s3 = storage + self.grace = grace + self.page_size = page_size + self.verify = verify + self.apply = apply + self.stats = Stats() + self.findings: list[Finding] = [] + + # ---------------------------------------------------------- streams + async def objects(self) -> AsyncIterator[ListedObject]: + after = None + while True: + page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size) + self.stats.s3_list_requests += 1 + for item in page: + self.stats.objects += 1 + yield item + if not more or not page: + return + after = page[-1].key + + async def rows(self) -> AsyncIterator[Row]: + after = "" + while True: + cur = await self.conn.execute( + "SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, " + "a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) " + "FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s", + (after, self.page_size)) + page = await cur.fetchall() + self.stats.db_pages += 1 + for r in page: + self.stats.rows += 1 + yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8]) + if len(page) < self.page_size: + return + after = page[-1][1] + + # ---------------------------------------------------------- planning (read-only) + async def scan(self) -> None: + cur = await self.conn.execute("SELECT clock_timestamp()") + self.db_now = (await cur.fetchone())[0] + self.s3_now = time.time() + objects, rows = self.objects(), self.rows() + obj = await anext(objects, None) + row = await anext(rows, None) + while obj is not None or row is not None: + if row is None or (obj is not None and obj.key < row.key): + self.object_only(obj) + obj = await anext(objects, None) + elif obj is None or row.key < obj.key: + self.row_only(row) + row = await anext(rows, None) + else: + await self.both(obj, row) + obj = await anext(objects, None) + row = await anext(rows, None) + + def row_old(self, row: Row) -> bool: + return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace + + def object_only(self, obj: ListedObject) -> None: + old = self.s3_now - obj.last_modified_ts > self.grace + self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key)) + + def row_only(self, row: Row) -> None: + if row.status == "pending": + self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id, + detail="object absent")) + elif row.lost_reason is not None: + self.stats.consistent += 1 # already quarantined, still missing: nothing new + else: + self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id, + used=row.used)) + + async def both(self, obj: ListedObject, row: Row) -> None: + if row.status == "pending": + self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id, + detail="object present")) + return + problem = await self.object_problem(row, obj.size) + if problem: + if row.lost_reason is None: + self.add(Finding(problem, row.key, row.id, used=row.used)) + else: + self.stats.consistent += 1 # quarantined and still broken + elif row.lost_reason == "checksum_mismatch" and not self.verify: + # Same size, but this pass does not read the bytes: it cannot prove the object is the + # original again, so a checksum quarantine is lifted only by --verify-checksums. + self.stats.consistent += 1 + elif row.lost_reason is not None: + self.add(Finding("restored_object", row.key, row.id, used=row.used)) + else: + self.stats.consistent += 1 + + async def object_problem(self, row: Row, size: int) -> str | None: + if size != row.size: + return "size_mismatch" + if self.verify: + data = await self.s3.read_all(row.key, row.size) + self.stats.s3_get_bytes += len(data) + if hashlib.sha256(data).digest() != row.sha256: + return "checksum_mismatch" + return None + + def add(self, finding: Finding) -> None: + if finding.kind not in ACTION_OF: + finding.result = "not_actionable" + self.findings.append(finding) + + # ---------------------------------------------------------- actions (--apply only) + async def run_actions(self, max_orphan_fraction: float) -> None: + orphans = sum(1 for f in self.findings if f.kind == "orphan_object") + if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects: + raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned " + f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed") + handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent, + "quarantine": self.quarantine, "unquarantine": self.unquarantine} + for finding in self.findings: + action = ACTION_OF.get(finding.kind) + if action is None: + continue + try: + finding.result = await handlers[action](finding) + except (Unavailable, psycopg.OperationalError) as exc: + finding.result = f"failed:{type(exc).__name__}" + raise # a store is down: stop; the next run continues from the committed state + + async def delete_object(self, f: Finding) -> str: + cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,)) + if await cur.fetchone(): + return "skipped:row_appeared" + try: + head = await self.s3.head(f.key) + self.stats.s3_head_requests += 1 + except ObjectMissing: + return "skipped:already_gone" + if time.time() - head.last_modified_ts <= self.grace: + return "skipped:object_rewritten_recently" + # No row names this key and the object is older than the grace period. Rows are always + # written BEFORE their object and keys are random and never reused, so no upload can + # still claim it. + await self.s3.delete(f.key) + return "applied" + + async def delete_intent(self, f: Finding) -> str: + async with self.conn.transaction(): + cur = await self.conn.execute( + "SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' " + "AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE", + (f.attachment_id, self.grace)) + if await cur.fetchone() is None: + return "skipped:no_longer_stale_intent" + await self.s3.delete(f.key) # object first (idempotent), then the row + await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'", + (f.attachment_id,)) + return "applied" + + async def quarantine(self, f: Finding) -> str: + async with self.conn.transaction(): + cur = await self.conn.execute( + "SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' " + "AND lost_at IS NULL FOR UPDATE", (f.attachment_id,)) + row = await cur.fetchone() + if row is None: + return "skipped:row_changed" + reason = await self.current_problem(f.key, row[0], bytes(row[1])) + if reason is None: + return "skipped:object_ok_now" + await self.conn.execute( + "UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s", + (reason, f.attachment_id)) + return "applied" + + async def unquarantine(self, f: Finding) -> str: + async with self.conn.transaction(): + cur = await self.conn.execute( + "SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE", + (f.attachment_id,)) + row = await cur.fetchone() + if row is None: + return "skipped:row_changed" + if row[2] == "checksum_mismatch" and not self.verify: + return "skipped:needs_verify_checksums" + if await self.current_problem(f.key, row[0], bytes(row[1])) is not None: + return "skipped:object_still_bad" + await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s", + (f.attachment_id,)) + return "applied" + + async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None: + """Fresh look at the object right before changing the row (the listing may be stale).""" + try: + head = await self.s3.head(key) + self.stats.s3_head_requests += 1 + except ObjectMissing: + return "missing_object" + if head.size != size: + return "size_mismatch" + if self.verify: + data = await self.s3.read_all(key, size) + self.stats.s3_get_bytes += len(data) + if hashlib.sha256(data).digest() != sha256: + return "checksum_mismatch" + return None + + # ---------------------------------------------------------- report + def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict: + kinds = Counter(f.kind for f in self.findings) + results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF) + samples = {} + for f in self.findings: + items = samples.setdefault(f.kind, []) + if len(items) < sample: + items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used, + "result": f.result, **({"detail": f.detail} if f.detail else {})}) + return { + "mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify, + "duration_seconds": round(seconds, 3), "error": error, + "scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent}, + "findings": {k: kinds.get(k, 0) for k in KINDS}, + "used_attachments_in_findings": sum(1 for f in self.findings if f.used), + "actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF), + "applied": results.get("applied", 0), "skipped": results.get("skipped", 0), + "failed": results.get("failed", 0)}, + "cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests, + "s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages, + "page_size": self.page_size}, + "samples": samples, + } + + +class SafetyValve(Exception): + pass + + +def arguments(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0]) + p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)") + p.add_argument("--grace", type=float, default=3600.0, + help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)") + p.add_argument("--allow-small-grace", action="store_true", + help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the " + "intent of an upload that is still running; that upload then fails with 503)") + p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)") + p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256") + p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report") + p.add_argument("--max-orphan-fraction", type=float, default=0.5, + help="--apply refuses if more than this share of objects look orphaned (default 0.5)") + args = p.parse_args(argv) + if not 1 <= args.page_size <= 1000: + p.error("--page-size must be 1..1000") + if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace): + p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)") + if not 0 < args.max_orphan_fraction <= 1: + p.error("--max-orphan-fraction must be in (0, 1]") + return args + + +async def run(args: argparse.Namespace) -> int: + url = _secret("DATABASE_URL", required=True) + base = s3_settings() + settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, + database_url=url, **base) + timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000) + options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}" + if not args.apply: + options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes + mode = "apply" if args.apply else "dry-run" + storage = Storage(settings) + await storage.open() + started = time.perf_counter() + error = None + code = 0 + try: + conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options, + application_name=f"messenger-reconcile-{mode}") + except psycopg.OperationalError as exc: + await storage.close() + error = f"store_unavailable: database {type(exc).__name__}" + print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False)) + log(error) + return 1 + rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace, + page_size=args.page_size, verify=args.verify_checksums, apply=args.apply) + try: + log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments") + await rec.scan() + log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings") + if args.apply: + await rec.run_actions(args.max_orphan_fraction) + except SafetyValve as exc: + error, code = f"safety_valve: {exc}", 3 + except (Unavailable, psycopg.OperationalError) as exc: + error, code = f"store_unavailable: {type(exc).__name__}", 1 + finally: + await conn.close() + await storage.close() + print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False, + indent=1)) + if error: + log(error) + return code + + +def main(argv: list[str] | None = None) -> int: + args = arguments(argv) + try: + return asyncio.run(run(args)) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/repo.py b/src/messenger/repo.py new file mode 100644 index 00000000..05791c0f --- /dev/null +++ b/src/messenger/repo.py @@ -0,0 +1,372 @@ +"""SQL for users, chats and messages. All statements are parameterised (%s placeholders): +values travel separately from the SQL text, so input can never change the query. + +Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is +encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md. +""" +from __future__ import annotations + +import datetime as dt +import uuid +from dataclasses import dataclass + +import psycopg +from psycopg import errors as pg_errors + +from .cursor import SortKey +from .db import Database +from .errors import Conflict, InvalidRequest, NotFound, Unavailable +from .timefmt import from_us, rfc3339, to_us + + +def _b(text: str) -> bytes: + return text.encode("utf-8") + + +def _s(raw: bytes | memoryview) -> str: + return bytes(raw).decode("utf-8") + + +@dataclass(slots=True, frozen=True) +class User: + id: str + username: str + display_name: str + created_at: dt.datetime + + def to_json(self) -> dict: + return {"id": self.id, "username": self.username, "display_name": self.display_name, + "created_at": rfc3339(self.created_at)} + + +@dataclass(slots=True, frozen=True) +class Chat: + id: str + title: str + member_ids: tuple[str, ...] + created_at: dt.datetime + + @property + def key(self) -> SortKey: + return (to_us(self.created_at), self.id) + + def to_json(self) -> dict: + return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids), + "created_at": rfc3339(self.created_at)} + + +@dataclass(slots=True, frozen=True) +class Message: + id: str + chat_id: str + sender_id: str + text: str + created_at: dt.datetime + attachment_ids: tuple[str, ...] = () + + @property + def key(self) -> SortKey: + return (to_us(self.created_at), self.id) + + def to_json(self) -> dict: + return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text, + "created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)} + + +def _user(row) -> User: + return User(str(row[0]), row[1], _s(row[2]), row[3]) + + +# ------------------------------------------------------------------ users +async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User: + async def work(conn: psycopg.AsyncConnection) -> User: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) " + "RETURNING id, username, display_name, created_at", + (uuid.uuid4(), username, _b(display_name), password_hash)) + return _user(await cur.fetchone()) + try: + return await db.run(work) + except pg_errors.UniqueViolation: + # Two concurrent registrations: the unique index lets exactly one INSERT commit. + raise Conflict("Имя пользователя уже занято") from None + + +async def get_user(db: Database, user_id: str) -> User | None: + async def work(conn: psycopg.AsyncConnection) -> User | None: + cur = await conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,)) + row = await cur.fetchone() + return _user(row) if row else None + return await db.run(work) + + +async def get_credentials(db: Database, username: str) -> tuple[User, str] | None: + async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None: + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,)) + row = await cur.fetchone() + return (_user(row), row[4]) if row else None + return await db.run(work) + + +# ------------------------------------------------------------------ chats +async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat: + """Atomic: the chat row and all memberships commit together or not at all. + + The chat is inserted FIRST; an unknown member then fails the membership INSERT + with a foreign-key violation and the whole transaction (chat row included) is + rolled back. student-tests check that nothing is left behind. + """ + if creator_id not in member_ids: + raise InvalidRequest("member_ids должен включать текущего пользователя") + chat_id = uuid.uuid4() + + async def work(conn: psycopg.AsyncConnection) -> dt.datetime: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at", + (chat_id, _b(title), creator_id)) + (created_at,) = await cur.fetchone() + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " + "SELECT %s, m.user_id, m.ord - 1, %s " + "FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)", + (chat_id, created_at, member_ids)) + return created_at + try: + created_at = await db.run(work) + except pg_errors.ForeignKeyViolation: + raise InvalidRequest("member_ids содержит несуществующих пользователей") from None + return Chat(str(chat_id), title, tuple(member_ids), created_at) + + +async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]: + if not rows: + return [] + ids = [r[0] for r in rows] + cur = await conn.execute( + "SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,)) + members: dict[uuid.UUID, list[str]] = {i: [] for i in ids} + for chat_id, user_id in await cur.fetchall(): + members[chat_id].append(str(user_id)) + return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows] + + +async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat: + """The chat if the user is a member; a missing chat and a foreign chat are both 404.""" + async def work(conn: psycopg.AsyncConnection) -> Chat: + cur = await conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id)) + row = await cur.fetchone() + if row is None: + raise NotFound("Чат не найден") + return (await _load_chats(conn, [row]))[0] + return await db.run(work) + + +async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None: + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)) + if await cur.fetchone() is None: + raise NotFound("Чат не найден") + + +async def ensure_member(db: Database, user_id: str, chat_id: str) -> None: + async def work(conn: psycopg.AsyncConnection) -> None: + await _check_member(conn, user_id, chat_id) + await db.run(work) + + +async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]: + async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]: + if after is None: + cur = await conn.execute( + "SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id " + "WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s", + (user_id, limit + 1)) + else: + cur = await conn.execute( + "SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id " + "WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) " + "ORDER BY m.chat_created_at, m.chat_id LIMIT %s", + (user_id, from_us(after[0]), after[1], limit + 1)) + rows = await cur.fetchall() + return await _load_chats(conn, rows[:limit]), len(rows) > limit + return await db.run(work) + + +# ------------------------------------------------------------------ messages +@dataclass(slots=True, frozen=True) +class PostResult: + message: Message + replayed: bool # True: the key was already used with this body, nothing new was written + + +async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str, + text: str, attachment_ids: list[str]) -> dt.datetime: + """Insert with a per-chat strictly increasing timestamp. + + `UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one + chat are serialised: a message with a smaller created_at can never commit after + a message with a bigger one, and a reader paging by cursor cannot skip it. + """ + cur = await conn.execute( + "UPDATE chats SET last_message_at = greatest(clock_timestamp(), " + "last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at", + (chat_id,)) + (created_at,) = await cur.fetchone() + await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) " + "VALUES (%s, %s, %s, %s, %s, %s::uuid[])", + (message_id, chat_id, user_id, _b(text), created_at, attachment_ids)) + if attachment_ids: + await conn.execute( + "INSERT INTO message_attachments (message_id, position, attachment_id) " + "SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)", + (message_id, attachment_ids)) + return created_at + + +# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409). +# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT. +CHECK_ATTACHMENTS = ( + "SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM (" + "SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' " + "FOR SHARE) a") + + +async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str: + if not attachment_ids: + return "ok" + cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id)) + found, usable = await cur.fetchone() + if found != len(attachment_ids): + return "attachment_not_found" + return "ok" if usable == found else "attachment_conflict" + + +def _attachment_error(status: str) -> None: + if status == "attachment_not_found": + raise NotFound("Вложение не найдено") + if status == "attachment_conflict": + raise Conflict("Вложение недоступно и не может быть прикреплено") + + +# Claim the key. A row of a concurrent transaction that is not committed yet makes this +# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict +# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is +# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired). +CLAIM_KEY = ( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) " + "VALUES (%s, %s, %s, %s, %s) " + "ON CONFLICT (user_id, chat_id, key) DO UPDATE SET " + "request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() " + "WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) " + "RETURNING 1") +STORED_RESULT = ( + "SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k " + "JOIN messages m ON m.id = k.message_id " + "WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s") + + +async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str, + ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult: + """Idempotent send: the key, the message and the stored result commit in ONE transaction. + + * first request -> key row + message + chat clock, all or nothing (no "pending" key is + ever left behind: a crash or 5xx rolls everything back and the client may retry); + * same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed; + * same key, other body -> 409; the membership check comes first, so outsiders always get 404; + * attachments (lab 4) are checked before the key is claimed (same order as post_message_v2). + """ + message_id = uuid.uuid4() + attachment_ids = list(attachment_ids) + + async def work(conn: psycopg.AsyncConnection) -> PostResult: + await _check_member(conn, user_id, chat_id) + check = await _check_attachments(conn, chat_id, attachment_ids) + if check == "ok": + cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds)) + if await cur.fetchone() is not None: + created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids) + return PostResult(Message(str(message_id), chat_id, user_id, text, created_at, + tuple(attachment_ids)), False) + cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key)) + row = await cur.fetchone() + if row is None: + _attachment_error(check) + # The key row vanished between the two statements (expired-key cleanup): retryable. + raise Unavailable("Повторите запрос с тем же Idempotency-Key") + if bytes(row[0]) != request_hash: + raise Conflict("Idempotency-Key уже использован с другим сообщением") + return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3], + tuple(str(a) for a in row[4])), True) + return await db.run(work) + + +POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments " + "FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])") + + +async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, + text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult: + """Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 = + 0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction.""" + message_id = uuid.uuid4() + + async def work(conn: psycopg.AsyncConnection) -> tuple: + cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text), + ttl_seconds, list(attachment_ids))) + return await cur.fetchone() + status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False) + if status == "not_found": + raise NotFound("Чат не найден") + _attachment_error(status) + if status == "conflict": + raise Conflict("Idempotency-Key уже использован с другим сообщением") + if status == "retry": + raise Unavailable("Повторите запрос с тем же Idempotency-Key") + return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at, + tuple(str(a) for a in stored_attachments)), status == "replayed") + + +async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int: + """Delete expired keys in bounded batches; one replica at a time (advisory xact lock).""" + async def work(conn: psycopg.AsyncConnection) -> int: + cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,)) + if not (await cur.fetchone())[0]: + return 0 + cur = await conn.execute( + "DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY(" + "SELECT ctid FROM idempotency_keys " + "WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))", + (ttl_seconds, batch)) + return cur.rowcount + return await db.run(work) + + +# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR +# would push the cursor into a Filter and late pages would scan the whole prefix. +# attachment_ids is a column of the message row (migration 0004): still one index range scan. +HISTORY_FIRST = ( + "SELECT id, sender_id, text, created_at, attachment_ids FROM messages " + "WHERE chat_id = %s ORDER BY created_at, id LIMIT %s") +HISTORY_AFTER = ( + "SELECT id, sender_id, text, created_at, attachment_ids FROM messages " + "WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s") + + +async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]: + async def work(conn: psycopg.AsyncConnection) -> list: + if after is None: + cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1)) + else: + cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1)) + return await cur.fetchall() + rows = await db.run(work) + items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4])) + for r in rows[:limit]] + return items, len(rows) > limit diff --git a/src/messenger/s3_bootstrap.py b/src/messenger/s3_bootstrap.py new file mode 100644 index 00000000..91a1b844 --- /dev/null +++ b/src/messenger/s3_bootstrap.py @@ -0,0 +1,151 @@ +"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`). + +Idempotent, safe to run on every `docker compose up`: + +1. waits until the S3 endpoint answers; +2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and + its objects are left untouched); +3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs; +4. proves the access model and FAILS the stand (exit 1) if it is wrong: + * anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused; + * the API key pair can read/write objects of ITS bucket, but can neither list it, nor create + a bucket, nor see or read another bucket (a canary bucket created here for this check); + * the reconciler key pair can list the bucket but cannot create one. +Only file names and verdicts are printed, never key values. +""" +from __future__ import annotations + +import asyncio +import os +import sys +import time +import urllib.error +import urllib.request + +from botocore.exceptions import ClientError + +from .config import ConfigError, Settings, _secret, s3_settings +from .errors import Unavailable +from .storage import SENTINEL_KEY, Storage + + +def say(text: str) -> None: + print(f"s3-bootstrap: {text}", flush=True) + + +def anonymous_status(url: str) -> int: + """HTTP status of an UNSIGNED request (what anyone on the network could do).""" + try: + with urllib.request.urlopen(url, timeout=3) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +async def error_code(coro) -> str: + """S3 error code of a call that is expected to be refused ('' if it succeeded).""" + try: + await coro + return "" + except ClientError as exc: + return str(exc.response.get("Error", {}).get("Code", "error")) + + +def settings_with(keys: tuple[str, str]) -> Settings: + base = s3_settings(credentials_required=False) + base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1]) + return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base) + + +async def run() -> int: + admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True)) + api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True)) + reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True)) + settings = settings_with(admin) + bucket = settings.s3_bucket + admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler)) + for s in (admin_s3, api_s3, rec_s3): + await s.open() + try: + deadline = time.monotonic() + 60 + while True: + try: + await admin_s3._call("list_buckets", admin_s3.client.list_buckets()) + break + except Unavailable: + if time.monotonic() > deadline: + say("S3 endpoint does not answer within 60 s") + return 1 + await asyncio.sleep(1) + + code = await error_code(admin_s3.client.head_bucket(Bucket=bucket)) + if code == "": + say(f"bucket {bucket}: exists, kept as is") + elif code in ("404", "NoSuchBucket", "NotFound"): + code = await error_code(admin_s3.client.create_bucket(Bucket=bucket)) + if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + say(f"bucket {bucket}: create failed ({code})") + return 1 + say(f"bucket {bucket}: created") + else: + say(f"bucket {bucket}: head failed ({code})") + return 1 + await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {}) + say(f"sentinel {SENTINEL_KEY}: written") + canary = bucket + "-canary" + if await error_code(admin_s3.client.head_bucket(Bucket=canary)): + await admin_s3.client.create_bucket(Bucket=canary) + await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket") + + failures = [] + base = settings.s3_endpoint.rstrip("/") + for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"), + ("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")): + status = anonymous_status(url) + verdict = "refused" if status in (401, 403) else "ALLOWED" + say(f"{what}: HTTP {status} -> {verdict}") + if verdict != "refused": + failures.append(what) + + probe = "health/bootstrap-probe" + checks = [ + ("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True), + ("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True), + ("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True), + ("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False), + ("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False), + ("api: read object of another bucket", + await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False), + ("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True), + ("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False), + ] + for what, code, allowed in checks: + ok = (code == "") == allowed + say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}") + if not ok: + failures.append(what) + visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])] + say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}") + if canary in visible: + failures.append("api sees another bucket") + if failures: + say("access model is WRONG: " + ", ".join(failures)) + return 1 + say("done: private bucket, least-privilege keys verified") + return 0 + finally: + for s in (admin_s3, api_s3, rec_s3): + await s.close() + + +def main() -> int: + try: + return asyncio.run(run()) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + os.environ.setdefault("PYTHONUNBUFFERED", "1") + sys.exit(main()) diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..53736e44 --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,160 @@ +"""Server-side sessions behind an opaque Bearer token. + +* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is + stored, so a leaked database/AOF does not contain usable tokens. A fast hash is + enough for a 256-bit random value; passwords need a slow KDF (passwords.py). +* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup + compares it with the current time, and a restart never extends it. + +Two interchangeable stores (SESSION_STORE): + +* `valkey` (compose default): the ONLY copy of a session is a Valkey key with + `PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second + copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked + token cannot come back after Valkey fails and recovers. If Valkey is unreachable + a session cannot be checked: protected requests and logout get 503, never access. +* `postgres`: table `sessions` (migrations/0001_init.sql). +""" +from __future__ import annotations + +import asyncio +import datetime as dt +import hashlib +import json +import logging +import secrets +from typing import Protocol + +from redis import exceptions as redis_errors +from redis.asyncio import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +from .config import Settings +from .db import Database +from .errors import Unavailable +from .timefmt import UTC, from_us, to_us + +log = logging.getLogger("messenger.sessions") + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() + + +def now() -> dt.datetime: + return dt.datetime.now(UTC) + + +class SessionStore(Protocol): + name: str + + async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ... + async def user_id_for(self, token: str) -> str | None: ... + async def revoke(self, token: str) -> bool: ... + async def ping(self) -> bool: ... + async def close(self) -> None: ... + + +class PostgresSessionStore: + name = "postgres" + + def __init__(self, db: Database): + self.db = db + + async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: + token = new_token() + expires_at = now() + dt.timedelta(seconds=ttl_seconds) + + async def work(conn) -> None: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash(token), user_id, expires_at)) + # Housekeeping: drop this user's expired sessions (bounded by the user's own rows). + await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now())) + await self.db.run(work) + return token, expires_at + + async def user_id_for(self, token: str) -> str | None: + async def work(conn): + cur = await conn.execute( + "SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now())) + return await cur.fetchone() + row = await self.db.run(work) + return str(row[0]) if row else None + + async def revoke(self, token: str) -> bool: + async def work(conn) -> bool: + cur = await conn.execute( + "DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now())) + return cur.rowcount == 1 + return await self.db.run(work) + + async def ping(self) -> bool: + return True # covered by the database readiness check + + async def close(self) -> None: + return None + + +class ValkeySessionStore: + name = "valkey" + PREFIX = "session:" + + def __init__(self, settings: Settings): + self.timeout = settings.cache_timeout_seconds + self.client = Redis.from_url( + settings.cache_url, + socket_timeout=self.timeout, + socket_connect_timeout=self.timeout, + retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once + decode_responses=False, + ) + + def _key(self, token: str) -> str: + return self.PREFIX + token_hash(token).hex() + + async def _call(self, coro): + try: + async with asyncio.timeout(self.timeout * 2): + return await coro + except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc: + log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}}) + raise Unavailable("Хранилище сессий временно недоступно") from None + + async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: + token = new_token() + expires_at = now() + dt.timedelta(seconds=ttl_seconds) + value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode() + # PXAT = absolute expiry time: survives restarts (AOF) without being extended. + await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000)) + return token, expires_at + + async def user_id_for(self, token: str) -> str | None: + raw = await self._call(self.client.get(self._key(token))) + if raw is None: + return None + data = json.loads(raw) + if from_us(data["expires_at_us"]) <= now(): + return None # do not rely only on Valkey's lazy expiry + return data["user_id"] + + async def revoke(self, token: str) -> bool: + return await self._call(self.client.delete(self._key(token))) == 1 + + async def ping(self) -> bool: + try: + return bool(await self._call(self.client.ping())) + except Unavailable: + return False + + async def close(self) -> None: + await self.client.aclose() + + +def build_store(settings: Settings, db: Database) -> SessionStore: + return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db) diff --git a/src/messenger/storage.py b/src/messenger/storage.py new file mode 100644 index 00000000..a8d7e6d0 --- /dev/null +++ b/src/messenger/storage.py @@ -0,0 +1,177 @@ +"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call. + +* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE -> + Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no + Admin (cannot create/delete buckets or read other buckets). The admin pair is given only + to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap). +* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a + paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a + libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled. +* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3 + restarts, pooled keep-alive connections to the old process are dead and the first request on + such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean + clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen + key, GET, HEAD, DELETE), so repeating it cannot create a second attachment. +* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way. + botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"): + plain, widely supported S3 requests. +""" +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import hashlib +import logging +from collections.abc import AsyncIterator +from dataclasses import dataclass + +from aiobotocore.config import AioConfig +from aiobotocore.session import get_session +from botocore.exceptions import BotoCoreError, ClientError + +from .config import Settings +from .errors import Unavailable + +log = logging.getLogger("messenger.storage") + +MISSING_CODES = {"NoSuchKey", "404", "NotFound"} + + +class ObjectMissing(Exception): + """The key does not exist in the bucket (definitive answer of S3, not an outage).""" + + +@dataclass(frozen=True, slots=True) +class ObjectHead: + size: int + content_type: str + metadata: dict[str, str] + last_modified_ts: float # POSIX seconds + + +@dataclass(frozen=True, slots=True) +class ListedObject: + key: str + size: int + last_modified_ts: float # POSIX seconds + + +class Storage: + def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None): + self.bucket = settings.s3_bucket + self.timeout = settings.s3_timeout_seconds + self._endpoint = settings.s3_endpoint + self._region = settings.s3_region + self._access_key = access_key or settings.s3_access_key_id + self._secret_key = secret_key or settings.s3_secret_access_key + self._config = AioConfig( + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_timeout_seconds, + retries={"total_max_attempts": 2, "mode": "standard"}, + s3={"addressing_style": "path"}, + signature_version="s3v4", + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + max_pool_connections=settings.s3_max_connections, + ) + self._stack: contextlib.AsyncExitStack | None = None + self._client = None + + async def open(self) -> None: + self._stack = contextlib.AsyncExitStack() + session = get_session() + self._client = await self._stack.enter_async_context(session.create_client( + "s3", endpoint_url=self._endpoint, region_name=self._region, + aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config)) + + async def close(self) -> None: + if self._stack is not None: + await self._stack.aclose() + self._stack = None + + @property + def client(self): + assert self._client is not None, "Storage.open() was not called" + return self._client + + async def _call(self, what: str, coro, deadline: float | None = None): + """Await one S3 call under the deadline; outages become Unavailable (503).""" + try: + return await asyncio.wait_for(coro, deadline or self.timeout) + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code in MISSING_CODES: + raise ObjectMissing() from None + log.warning("s3_error", extra={"fields": {"op": what, "code": code}}) + raise Unavailable("Хранилище файлов временно недоступно") from None + except (asyncio.TimeoutError, BotoCoreError, OSError) as exc: + log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}}) + raise Unavailable("Хранилище файлов временно недоступно") from None + + async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None: + md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii") + # Time budget grows with the size: 10 MiB over a local network takes well under a second. + deadline = self.timeout + len(data) / (8 * 1024 * 1024) + await self._call("put", self.client.put_object( + Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data), + ContentMD5=md5, Metadata=metadata), deadline) + + async def head(self, key: str) -> ObjectHead: + response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key)) + return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""), + dict(response.get("Metadata", {})), response["LastModified"].timestamp()) + + async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]: + """Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the + deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the + download instead of hanging the connection).""" + response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key)) + body = response["Body"] + size = int(response["ContentLength"]) + + async def chunks() -> AsyncIterator[bytes]: + try: + while True: + chunk = await self._call("get_body", body.read(64 * 1024)) + if not chunk: + return + yield chunk + finally: + body.close() + return size, chunks() + + async def read_all(self, key: str, limit: int) -> bytes: + """Whole object (reconciler checksum verification); refuses objects above `limit`.""" + size, chunks = await self.open_stream(key) + if size > limit: + raise Unavailable("Объект больше допустимого размера") + parts = [chunk async for chunk in chunks] + return b"".join(parts) + + async def delete(self, key: str) -> None: + """Idempotent: deleting a missing key is a success in S3.""" + await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key)) + + async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]: + """One ListObjectsV2 page in key order (needs the List permission: reconciler only).""" + params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size} + if start_after: + params["StartAfter"] = start_after + response = await self._call("list", self.client.list_objects_v2(**params)) + items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp()) + for o in response.get("Contents", [])] + return items, bool(response.get("IsTruncated")) + + async def ping(self, timeout: float = 1.0) -> bool: + """Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3 + answers, the bucket exists and the API credentials are accepted (Read permission).""" + try: + await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout) + return True + except (Unavailable, ObjectMissing): + return False + + +SENTINEL_KEY = "health/sentinel" +OBJECT_PREFIX = "o/" # every attachment original lives under o/ diff --git a/src/messenger/timefmt.py b/src/messenger/timefmt.py new file mode 100644 index 00000000..9c33b12f --- /dev/null +++ b/src/messenger/timefmt.py @@ -0,0 +1,22 @@ +"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings.""" +from __future__ import annotations + +import datetime as dt + +UTC = dt.timezone.utc +_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC) + + +def to_us(value: dt.datetime) -> int: + """Microseconds since the Unix epoch (exact: timestamptz has microsecond precision).""" + delta = value - _EPOCH + return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds + + +def from_us(us: int) -> dt.datetime: + return _EPOCH + dt.timedelta(microseconds=us) + + +def rfc3339(value: dt.datetime) -> str: + """RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z.""" + return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 00000000..3ad26ece --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,199 @@ +"""Strict request validation according to contracts/openapi.json + contracts/README.md. + +Written by hand instead of a schema library so that every rule is visible: +extra fields, wrong JSON types, whitespace-only strings and lengths counted in +Unicode code points all produce 400 invalid_request. +""" +from __future__ import annotations + +import json +import re +import unicodedata +from typing import Any +from urllib.parse import parse_qsl + +from .errors import InvalidRequest + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case. +UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +LIMIT_RE = re.compile(r"[0-9]{1,3}") +# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") +SURROGATES_RE = re.compile("[\ud800-\udfff]") + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 + + +def _reject_constant(name: str) -> Any: + # Python's json accepts NaN/Infinity, which are not valid JSON. + raise ValueError(f"invalid JSON constant {name}") + + +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + # {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins". + result = dict(pairs) + if len(result) != len(pairs): + raise ValueError("duplicate key in JSON object") + return result + + +def parse_json_object(body: bytes) -> dict[str, Any]: + """Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400).""" + try: + text = body.decode("utf-8") + value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys) + except (UnicodeDecodeError, ValueError): + raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None + if not isinstance(value, dict): + raise InvalidRequest("Тело запроса должно быть JSON-объектом") + return value + + +def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None: + extra = set(data) - required - optional + if extra: + raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra))) + missing = required - set(data) + if missing: + raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing))) + + +def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str: + value = data[name] + if not isinstance(value, str): + raise InvalidRequest(f"Поле {name} должно быть строкой") + if SURROGATES_RE.search(value): + # "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8. + raise InvalidRequest(f"Поле {name} содержит недопустимые символы") + # len() of a Python str is the number of Unicode code points, as the contract requires. + if not min_len <= len(value) <= max_len: + raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов") + if value.strip() == "" and not allow_blank: + raise InvalidRequest(f"Поле {name} не может состоять только из пробелов") + return value + + +def username_field(data: dict[str, Any]) -> str: + value = data["username"] + if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None: + raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}") + return value + + +def password_field(data: dict[str, Any]) -> str: + """12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed.""" + value = data["password"] + if not isinstance(value, str): + raise InvalidRequest("Поле password должно быть строкой") + if SURROGATES_RE.search(value): + raise InvalidRequest("Поле password содержит недопустимые символы") + if not 12 <= len(value) <= 128: + raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов") + if value.strip() == "": + raise InvalidRequest("Пароль не может состоять только из пробелов") + return value + + +def parse_uuid(value: Any, what: str = "UUID") -> str: + if not isinstance(value, str) or UUID_RE.fullmatch(value) is None: + raise InvalidRequest(f"Неверный {what}") + return value.lower() + + +def member_ids_field(data: dict[str, Any]) -> list[str]: + value = data["member_ids"] + if not isinstance(value, list): + raise InvalidRequest("member_ids должен быть массивом") + if not 1 <= len(value) <= 100: + raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов") + ids = [parse_uuid(item, "UUID участника") for item in value] + if len(set(ids)) != len(ids): + raise InvalidRequest("member_ids содержит дубликаты") + return ids + + +def attachment_ids_field(data: dict[str, Any]) -> list[str]: + """Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md); + null or any other non-array type is 400.""" + value = data.get("attachment_ids", []) + if not isinstance(value, list): + raise InvalidRequest("attachment_ids должен быть массивом") + if len(value) > 10: + raise InvalidRequest("attachment_ids содержит больше 10 элементов") + ids = [parse_uuid(item, "UUID вложения") for item in value] + if len(set(ids)) != len(ids): + raise InvalidRequest("attachment_ids содержит дубликаты") + return ids + + +def idempotency_key(raw: str | None) -> str: + if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None: + raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}") + return raw + + +def single_query_param(query: list[tuple[str, str]], name: str) -> str | None: + values = [v for k, v in query if k == name] + if len(values) > 1: + raise InvalidRequest(f"Параметр {name} указан несколько раз") + return values[0] if values else None + + +def parse_limit(raw: str | None) -> int: + if raw is None: + return DEFAULT_LIMIT + if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT: + raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}") + return int(raw) + + +# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown +# to other users, so characters that change how a name LOOKS are refused: control characters +# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls +# ("photognp.exe" would display as "photoexe.png"). +BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") +FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"}) + + +def query_items(raw: bytes) -> list[tuple[str, str]]: + """Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD).""" + try: + return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict") + except (UnicodeDecodeError, ValueError): + raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None + + +def filename_param(query: list[tuple[str, str]]) -> str: + value = single_query_param(query, "filename") + if value is None: + raise InvalidRequest("Параметр filename обязателен") + if not 1 <= len(value) <= 255: + raise InvalidRequest("Длина filename должна быть от 1 до 255 символов") + if value.strip() == "" or value in {".", ".."}: + raise InvalidRequest("filename не может состоять только из пробелов или точек") + if "/" in value or "\\" in value: + raise InvalidRequest("filename не может содержать / или \\") + for char in value: + if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS: + raise InvalidRequest("filename содержит управляющие символы") + return value + + +ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4 + + +def variant_param(query: list[tuple[str, str]]) -> str: + """`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant + name is a missing variant of a ready object (404); anything else is an invalid value (400).""" + value = single_query_param(query, "variant") + if value is None or value == "original": + return "original" + if value in ATTACHMENT_VARIANTS: + raise NotFoundVariant() + raise InvalidRequest("variant должен быть original") + + +class NotFoundVariant(Exception): + """Raised for a variant that lab 4 does not produce; mapped to 404 by the handler.""" diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..e1efdbb0 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,380 @@ +"""Shared helpers for the student tests. + +* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services. +* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose + stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus + `docker compose` for database checks (psql inside the postgres container), the HAProxy + runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the + stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP. + + python -m pytest student-tests # everything (stand must be up) + python -m pytest student-tests -m "not stack" # unit only +""" +from __future__ import annotations + +import base64 +import contextlib +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import quote + +import httpx2 as httpx +import pytest +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) +import lbctl # noqa: E402 + +SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8")) +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose +REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs + + +def pytest_configure(config): + config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)") + + +def schema_errors(value, name: str) -> list[str]: + """Validate against components/schemas/ of the course contract (JSON Schema 2020-12).""" + schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]} + validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER) + return [e.message for e in validator.iter_errors(value)] + + +def basic(username: str, password: str) -> str: + return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() + + +# ------------------------------------------------------------------ docker / psql +def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, + encoding="utf-8", check=check, timeout=timeout, input=stdin) + + +def psql(sql: str) -> str: + """Run SQL inside the postgres container (no DB port is published) and return unaligned output.""" + result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-v", "ON_ERROR_STOP=1", "-c", sql) + return result.stdout.strip() + + +def wait_ready(timeout: float = 90) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.3) + raise AssertionError("stand did not become ready") + + +def wait_healthy(*services: str, timeout: float = 120) -> None: + """Poll Docker health until EVERY container of every service is healthy again. `compose up + --wait` is not used: it gives up at once on a container marked unhealthy while paused.""" + deadline = time.monotonic() + timeout + pending = set(services) + while pending and time.monotonic() < deadline: + for service in list(pending): + ids = compose("ps", "-q", service, check=False).stdout.split() + if ids: + states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids], + capture_output=True, text=True).stdout.split() + if states and all(state == "healthy" for state in states): + pending.discard(service) + time.sleep(0.5) + assert not pending, f"not healthy: {pending}" + + +def wait_stand(replicas: int = REPLICAS) -> None: + """Readiness through the LB AND every replica back in HAProxy rotation.""" + wait_ready() + lbctl.wait_up(replicas, timeout=90) + + +def restart_services(*services: str) -> None: + compose("restart", *services) + wait_healthy(*services) + wait_stand() + + +@contextlib.contextmanager +def service_down(service: str, how: str = "stop"): + """Stop or pause a compose service; ALWAYS bring it back and wait for readiness.""" + assert how in {"stop", "pause"} + compose(how, service) + try: + yield + finally: + compose("unpause" if how == "pause" else "start", service) + wait_healthy(service) + wait_stand() + + +@contextlib.contextmanager +def api_with_env(**overrides: str): + """Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2), + then ALWAYS recreate it with the normal configuration.""" + env = {**os.environ, **overrides} + subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env, + capture_output=True, text=True, check=True, timeout=180) + wait_stand() + try: + yield + finally: + clean = {k: v for k, v in os.environ.items() if k not in overrides} + subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean, + capture_output=True, text=True, check=True, timeout=180) + wait_stand() + + +def running_replicas() -> list[dict]: + return [r for r in lbctl.replicas() if r["state"] == "running"] + + +@contextlib.contextmanager +def replica_stopped(replica: dict, mode: str = "stop"): + """Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again + and wait until HAProxy has every replica UP.""" + assert mode in {"stop", "kill"} + subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]], + capture_output=True, text=True, check=True, timeout=60) + try: + yield + finally: + subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60) + wait_healthy("api") + wait_stand() + + +def api_env(name: str) -> str: + """Current value of an environment variable inside the running api container.""" + return compose("exec", "-T", "api", "printenv", name).stdout.strip() + + +# ------------------------------------------------------------------ HTTP +class Api: + """HTTP helper that checks the invariants every response must satisfy.""" + + def __init__(self, base_url: str = BASE_URL): + self.client = httpx.Client(base_url=base_url, timeout=15) + + def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None, + json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None, + lb_error_ok: bool = False): + """lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is + accepted without X-Instance-Id; every response of an API replica must carry it.""" + h = dict(headers or {}) + if token is not None: + h["Authorization"] = "Bearer " + token + if json_body is not None: + raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8") + h.setdefault("Content-Type", "application/json") + response = self.client.request(method, path, content=raw, headers=h) + allowed = (expected,) if isinstance(expected, int) else expected + assert response.status_code in allowed, (method, path, response.status_code, response.text[:300]) + request_id = response.headers.get("x-request-id") + assert request_id, "every response must carry X-Request-Id" + if not response.headers.get("x-instance-id"): + assert lb_error_ok and response.status_code in (502, 503, 504), \ + ("missing X-Instance-Id", method, path, response.status_code) + if response.status_code >= 400: + body = response.json() + assert schema_errors(body, "Error") == [], response.text + assert body["error"]["request_id"] == request_id + assert "Traceback" not in response.text and "psycopg" not in response.text + if response.status_code == 401: + challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer" + assert response.headers.get("www-authenticate", "").lower().startswith(challenge) + elif schema: + assert schema_errors(response.json(), schema) == [], response.text + return response + + def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict: + username = f"{label}_{uuid.uuid4().hex[:12]}"[:32] + body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password} + user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json() + return {"id": user["id"], "user": user, "username": username, "password": password} + + def login(self, actor: dict) -> str: + session = self.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(actor["username"], actor["password"])}, + schema="Session").json() + assert session["user"] == actor["user"] and session["token_type"] == "Bearer" + return session["token"] + + def user(self, label: str = "u", **kwargs) -> dict: + actor = self.register(label, **kwargs) + actor["token"] = self.login(actor) + return actor + + def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict: + ids = [owner["id"], *(m["id"] for m in members)] + return self.call("POST", "/api/v1/chats", 201, token=owner["token"], + json_body={"title": title, "member_ids": ids}, schema="Chat").json() + + def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None, + raw: bytes | None = None, headers: dict | None = None, **kwargs): + """POST a message with an Idempotency-Key (a fresh one unless `key` is given).""" + h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})} + if raw is not None: + h.setdefault("Content-Type", "application/json") + return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"], + json_body=body if raw is None else None, raw=raw, headers=h, + schema="Message" if expected == 201 else None, **kwargs) + + def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict: + return self.post_message(actor, chat, {"text": text}, key=key).json() + + def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png", + filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None): + """POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string + (to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked).""" + h = dict(headers or {}) + if content_type is not None: + h["Content-Type"] = content_type + if query is None: + query = "" if filename is None else "filename=" + quote(filename, safe="") + path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "") + if isinstance(data, (bytes, bytearray)): + return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h, + schema="Attachment" if expected == 201 else None) + h["Authorization"] = "Bearer " + actor["token"] + response = self.client.post(path, content=data, headers=h) + assert response.status_code == expected, (response.status_code, response.text[:300]) + return response + + def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: + """Walk all pages; returns (items, number_of_pages).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "") + page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200, + token=actor["token"], schema="MessagePage").json() + pages += 1 + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items, pages + assert len(page["items"]) == limit, "a non-final page must be full" + assert len(items) < 100_000, "cursor loop" + + +def _stand_is_up() -> bool: + try: + return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200 + except httpx.HTTPError: + return False + + +def pytest_collection_modifyitems(config, items): + if any("stack" in item.keywords for item in items) and not _stand_is_up(): + skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait") + for item in items: + if "stack" in item.keywords: + item.add_marker(skip) + + +@pytest.fixture(scope="session") +def api() -> Api: + return Api() + + +def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess: + """Run scripts/ (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers.""" + script = str(ROOT / "scripts" / name) + if os.name == "nt": + command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script] + else: + command = [script] + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True, + timeout=timeout, env={**os.environ, **env}) + + +def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str: + """Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3 + credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is + not published on the host; this is how the tests inject faults directly into the bucket.""" + script = ("import asyncio, base64, json, sys\n" + "from messenger.config import Settings, s3_settings\n" + "from messenger.storage import Storage, ObjectMissing\n" + "async def main():\n" + " st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n" + " s = Storage(st)\n" + " await s.open()\n" + " try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) + + " finally:\n" + " await s.close()\n" + "asyncio.run(main())\n") + if service == "api": + cmd = ["exec", "-T", "api", "python", "-c", script] + else: + cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script] + return compose(*cmd, timeout=120, stdin=stdin).stdout.strip() + + +def s3_put(key: str, data: bytes) -> None: + """Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB).""" + s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})", + stdin=base64.b64encode(data).decode()) + + +def s3_delete(key: str) -> None: + s3_exec(f"await s.delete({key!r})") + + +def s3_size(key: str) -> int | None: + out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)") + size = int(out.splitlines()[-1]) + return None if size < 0 else size + + +def s3_keys() -> set[str]: + """Every object key under o/ (needs List: runs with the reconciler's key pair).""" + out = s3_exec("keys, after = [], None\n" + "while True:\n" + " page, more = await s.list_page('o/', after, 1000)\n" + " keys += [o.key for o in page]\n" + " if not more or not page:\n" + " break\n" + " after = page[-1].key\n" + "print(json.dumps(keys))", service="reconciler") + return set(json.loads(out.splitlines()[-1])) + + +def reconcile(*args: str, expect_code: int = 0) -> dict: + """Run the reconciler CLI (compose service, profile tools) and return its JSON report.""" + # --no-deps: never let `compose run` start a store that a test has stopped on purpose. + result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False, + timeout=300) + assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:]) + return json.loads(result.stdout) + + +def findings_by_key(report: dict) -> dict[str, dict]: + return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items} + + +def attachment_row(attachment_id: str) -> dict | None: + out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, " + "'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id) + return json.loads(out) if out else None + + +def probe_inside(replica: dict, path: str) -> int: + """HTTP status of `path` asked inside the replica container itself (bypassing the LB).""" + code = ("import urllib.request as u, urllib.error as e\n" + "try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n" + "except e.HTTPError as x: print(x.code)") % path + out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True, + timeout=30).stdout.strip() + return int(out or 0) diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..0026fea8 --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,93 @@ +"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the +reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without +scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file +is generated in memory.""" +from __future__ import annotations + +import io +import os +import struct +import zlib + +from PIL import Image + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def chunk(ctype: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data)) + + +def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes: + return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)) + + +def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes: + if noise: + channels = len(mode) + image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels)) + else: + image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128) + out = io.BytesIO() + image.save(out, "PNG", **save) + return out.getvalue() + + +def png_of_size(target: int) -> bytes: + """A valid PNG of (about, at most) `target` bytes: random pixels do not compress.""" + side = int((target / 3) ** 0.5) + data = png(side, side, noise=True, compress_level=1) + while len(data) > target: + side -= 8 + data = png(side, side, noise=True, compress_level=1) + return data + + +def png_exactly(size: int) -> bytes: + """A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk.""" + base = png_of_size(size - 64) + pad = size - len(base) - 12 + assert pad >= 0 + return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:] + + +def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes: + if noise: + image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)) + else: + image = Image.new("RGB", (width, height), (200, 30, 30)) + out = io.BytesIO() + image.save(out, "JPEG", progressive=progressive, quality=quality) + return out.getvalue() + + +# ------------------------------------------------------------------ broken files +def png_corrupt_idat() -> bytes: + """Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case).""" + return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"") + + +def png_short_pixels() -> bytes: + """A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises.""" + raw = b"".join(b"\x00" + bytes(60) for _ in range(19)) + return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes: + """Header of a huge picture: must be refused (413) from the first 33 bytes.""" + raw_row = b"\x00" + bytes(width * 3) + body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far + return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"") + + +def jpeg_without_scan_data() -> bytes: + data = jpeg(128, 96) + sos = data.index(b"\xff\xda") + length = struct.unpack(">H", data[sos + 2:sos + 4])[0] + return data[:sos + 2 + length] + b"\xff\xd9" + + +def jpeg_bomb_header() -> bytes: + """SOF0 announcing 20000 x 20000 = 400 Mpx.""" + return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000) + + b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9") diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..13fc2d2e --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,344 @@ +"""Contract behaviour over real HTTP (stand must be running). Lab 1 checks moved to Bearer + new lab 2 auth.""" +from __future__ import annotations + +import base64 +import uuid + +import pytest + +from conftest import PASSWORD, basic + +pytestmark = pytest.mark.stack + + +def key(item: dict) -> tuple[str, int]: + return item["created_at"], uuid.UUID(item["id"]).int + + +# ================================================================== registration & login +def test_register_never_returns_password_and_login_returns_bearer(api): + actor = api.register("alice", display_name="Алиса 👩‍💻") + assert set(actor["user"]) == {"id", "username", "display_name", "created_at"} + session = api.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(actor["username"], PASSWORD)}, schema="Session").json() + assert session["token_type"] == "Bearer" and len(session["token"]) >= 43 + assert actor["username"] not in session["token"] and actor["id"] not in session["token"] + assert api.call("GET", "/api/v1/users/me", 200, token=session["token"], schema="User").json() == actor["user"] + + +def test_duplicate_username_is_409(api): + actor = api.register("dup") + body = {"username": actor["username"], "display_name": "Другой", "password": PASSWORD} + assert api.call("POST", "/api/v1/users", 409, json_body=body).json()["error"]["code"] == "conflict" + + +@pytest.mark.parametrize("password", ["short-11ch", " " * 12, " \t \n " * 3, "я" * 129]) +def test_bad_password_on_registration_is_400(api, password): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": password} + api.call("POST", "/api/v1/users", 400, json_body=body) + + +@pytest.mark.parametrize("password", ["я" * 128, "with:colons:inside", "Twelve chars"]) +def test_edge_passwords_register_and_login(api, password): + actor = api.user("edge", password=password) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_wrong_password_and_unknown_user_give_the_same_401(api): + actor = api.register("victim") + wrong = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(actor["username"], "Wrong-password-1")}) + unknown = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("nobody_" + uuid.uuid4().hex[:8], PASSWORD)}) + assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"] + assert wrong.headers["www-authenticate"] == 'Basic realm="messenger", charset="UTF-8"' + + +@pytest.mark.parametrize("header", ["Basic é", "Basic !!!", "Basic " + base64.b64encode(b"no_colon").decode(), + "Basic " + base64.b64encode(b"\xff\xfe:x").decode(), "Bearer " + "a" * 43, ""]) +def test_malformed_basic_is_401_not_500(api, header): + # Raw UTF-8 bytes, as curl would send them (httpx refuses non-ASCII str headers). + api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": header.encode("utf-8")} if header else {}) + + +def test_logout_revokes_only_this_token(api): + actor = api.user("logout") + second = api.login(actor) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"]) + api.call("GET", "/api/v1/users/me", 401, token=actor["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=actor["token"]) + api.call("GET", "/api/v1/users/me", 200, token=second) # other session of the same user survives + + +def test_x_user_id_basic_and_garbage_do_not_authenticate(api): + actor = api.user("hdr") + for headers in ({"X-User-Id": actor["id"]}, + {"Authorization": basic(actor["username"], PASSWORD)}, + {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}): + api.call("GET", "/api/v1/users/me", 401, headers=headers) + me = api.call("GET", "/api/v1/users/me", 200, token=actor["token"], headers={"X-User-Id": str(uuid.uuid4())}) + assert me.json()["id"] == actor["id"] + + +# ================================================================== validation / errors (from lab 1) +@pytest.mark.parametrize("raw", [b"{", b"", b"[]", b"null", b'{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}', + b'{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}', + b'{"username": "abc_def", "display_name": "\\ud800", "password": "Twelve-chars"}']) +def test_malformed_json_is_400(api, raw): + api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"}) + + +@pytest.mark.parametrize("body", [ + {"username": "abc_def", "display_name": "x"}, # no password + {"username": "abc_def", "display_name": "x", "password": "Twelve-chars", "extra": 1}, + {"username": "Abc_def", "display_name": "x", "password": "Twelve-chars"}, + {"username": "abc_def", "display_name": " ", "password": "Twelve-chars"}, + {"username": "abc_def", "display_name": "x" * 101, "password": "Twelve-chars"}, + {"username": 5, "display_name": "x", "password": "Twelve-chars"}, +]) +def test_invalid_registration_fields_are_400(api, body): + api.call("POST", "/api/v1/users", 400, json_body=body) + + +def test_unicode_limits_are_code_points_and_nul_is_stored(api): + actor = api.user("uni", display_name="😀" * 100) + assert actor["user"]["display_name"] == "😀" * 100 + nul = api.user("nul", display_name="A\u0000B") + assert api.call("GET", "/api/v1/users/me", 200, token=nul["token"]).json()["display_name"] == "A\u0000B" + chat = api.chat(actor, title="T\u0000itle") + assert chat["title"] == "T\u0000itle" + assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world" + assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000 + api.post_message(actor, chat, {"text": "я" * 4001}, 400) + texts = [m["text"] for m in api.history(actor, chat, 100)[0]] + assert texts == ["hello\u0000world", "🙂" * 4000] + + +def test_oversized_body_and_wrong_content_type(api): + api.call("POST", "/api/v1/users", 413, json_body={"username": "big_body", "display_name": "x" * 70_000, "password": PASSWORD}) + api.call("POST", "/api/v1/users", 400, raw=b'{"username":"abc_def"}', headers={"Content-Type": "text/plain"}) + + +# ================================================================== chats & membership +def test_chat_validation(api): + alice, bob = api.user("alice"), api.user("bob") + for body in ({"title": "x", "member_ids": [alice["id"], bob["id"], bob["id"]]}, + {"title": "x", "member_ids": [alice["id"], str(uuid.uuid4())]}, + {"title": "x", "member_ids": [bob["id"]]}, + {"title": " ", "member_ids": [alice["id"]]}, + {"title": "x", "member_ids": [alice["id"], "bob"]}): + api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body=body) + + +def test_foreign_chat_is_indistinguishable_from_missing(api): + alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") + chat = api.chat(alice, bob) + for chat_id in (chat["id"], str(uuid.uuid4())): + base = f"/api/v1/chats/{chat_id}" + errors = [api.call("GET", base, 404, token=eve["token"]).json(), + api.call("GET", base + "/messages", 404, token=eve["token"]).json(), + api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(), + api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()] + assert len({e["error"]["message"] for e in errors}) == 1 + api.call("GET", "/api/v1/chats/not-a-uuid", 401) + api.call("GET", "/api/v1/chats/not-a-uuid", 400, token=alice["token"]) + api.call("GET", "/api/v1/chats/NOT-A-UUID/messages", 400, token=alice["token"]) + + +def test_chat_list_order_pagination_and_isolation(api): + alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") + created = [api.chat(alice, bob, title=f"Чат {i}") for i in range(5)] + api.chat(eve, title="Чат Евы") + walked, cursor = [], None + while True: + query = "?limit=2" + (f"&cursor={cursor}" if cursor else "") + page = api.call("GET", "/api/v1/chats" + query, 200, token=bob["token"], schema="ChatPage").json() + walked += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + assert walked == created + first = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"] + api.call("GET", f"/api/v1/chats?limit=1&cursor={first}", 400, token=bob["token"]) # cursor of another user + api.call("GET", "/api/v1/chats?limit=0", 400, token=bob["token"]) + + +# ================================================================== messages & keyset pagination +def test_history_pages_without_loss_or_duplicates(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + sent = [api.send(alice if i % 2 else bob, chat, f"сообщение {i}") for i in range(25)] + items, pages = api.history(bob, chat, 7) + assert pages == 4 and items == sent and items == sorted(items, key=key) + assert sent[0]["sender_id"] == bob["id"] and sent[1]["sender_id"] == alice["id"] + + +@pytest.mark.parametrize("query", ["limit=0", "limit=101", "limit=abc", "limit=1&limit=2", "cursor=", "cursor=garbage", + "cursor=djF8bXx4fDF8Mg", "cursor=" + base64.urlsafe_b64encode(b"v1|m|x|99999999999999999999|y").decode()]) +def test_bad_limit_or_cursor_is_400(api, query): + alice = api.user("alice") + chat = api.chat(alice) + api.call("GET", f"/api/v1/chats/{chat['id']}/messages?{query}", 400, token=alice["token"]) + + +def test_cursor_bound_to_chat(api): + alice = api.user("alice") + a, b = api.chat(alice, title="A"), api.chat(alice, title="B") + for chat in (a, b): + api.send(alice, chat, "1") + api.send(alice, chat, "2") + cursor = api.call("GET", f"/api/v1/chats/{a['id']}/messages?limit=1", 200, token=alice["token"]).json()["next_cursor"] + api.call("GET", f"/api/v1/chats/{b['id']}/messages?cursor={cursor}", 400, token=alice["token"]) + empty = api.chat(alice, title="Empty") + api.call("GET", f"/api/v1/chats/{empty['id']}/messages?cursor=garbage", 400, token=alice["token"]) + + +def test_read_after_write(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + for i in range(5): + message = api.send(alice, chat, f"m{i}") + last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1] + assert last == message + + +# ================================================================== lab 3: inputs found by the reviewer in other PRs +def b64(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def bad_cursors(owner_id: str) -> list[str]: + """Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time.""" + good_uuid = str(uuid.uuid4()) + return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"), + b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'), + b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()), + b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()), + b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()), + b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()), + b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())] + + +def test_malformed_cursor_values_are_400_on_both_lists(api): + alice = api.user("cur") + chat = api.chat(alice) + for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]): + quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20") + for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json() + assert error["error"]["code"] == "invalid_request", (path, cursor) + + +@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}', + b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}', + b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}', + b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}', + b'{"text":" \n\t "}']) +def test_bad_message_body_is_400_and_nothing_is_stored(api, raw): + alice = api.user("body") + chat = api.chat(alice) + error = api.post_message(alice, chat, None, 400, raw=raw).json() + assert error["error"]["code"] == "invalid_request" + assert api.history(alice, chat, 100)[0] == [] + + +def test_ff_byte_body_on_registration_is_contract_error(api): + response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"}) + assert response.json()["error"]["request_id"] == response.headers["x-request-id"] + + +def test_text_is_stored_verbatim_and_length_counts_raw_text(api): + alice = api.user("verbatim") + chat = api.chat(alice) + padded = " строка\nс пробелами " + assert api.send(alice, chat, padded)["text"] == padded + api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming + assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded] + + +def test_unknown_attachment_ids_are_404(api): + alice = api.user("att") + chat = api.chat(alice) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates + assert api.history(alice, chat, 100)[0] == [] + + +def test_member_ids_are_compared_as_uuids_not_strings(api): + alice, bob = api.user("case"), api.user("case") + api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]}) + api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())}) + chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json() + assert chat["member_ids"] == [alice["id"], bob["id"]] + + +@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43]) +def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header): + api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")}) + + +def test_login_with_a_body_is_rejected(api): + actor = api.register("login_body") + auth = {"Authorization": basic(actor["username"], actor["password"])} + api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"}) + api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth) + api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session") + + +def test_password_range_is_12_to_128_code_points(api): + api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"}) + actor = api.user("pw128", password="я:" * 64) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_every_api_response_carries_instance_and_request_id(api): + """X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it).""" + api.call("GET", "/health/live", 200) + api.call("GET", "/health/ready", 200) + api.call("GET", "/no/such/route", 404) + api.call("PUT", "/api/v1/users", 405) + api.call("GET", "/api/v1/users/me", 401) + response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"}) + assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log + + +@pytest.mark.parametrize("blank", [" ", "  ", "  \t "]) +def test_unicode_whitespace_only_strings_are_400(api, blank): + alice = api.user("blank") + chat = api.chat(alice) + api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank, + "password": PASSWORD}) + api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]}) + api.post_message(alice, chat, {"text": blank}, 400) + assert api.history(alice, chat, 100)[0] == [] + + +@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}', + '{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}', + b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}']) +def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw): + api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"}) + api.call("GET", "/health/ready", 200) + + +def test_more_than_100_members_is_400(api): + owner = api.user("big") + others = [api.register(f"m{i}")["id"] for i in range(100)] + api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]}) + chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"], + json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json() + assert len(chat["member_ids"]) == 100 + + +def test_cursor_of_one_list_is_rejected_by_the_other(api): + alice = api.user("cross") + chats = [api.chat(alice, title=f"c{i}") for i in range(2)] + for _ in range(2): + api.send(alice, chats[0], "m") + message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200, + token=alice["token"]).json()["next_cursor"] + chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"] + api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"]) + api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..8aaa0157 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,268 @@ +"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits, +content checks, filename rules and messages with attachment_ids. + +Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted +IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000 +header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace +filename, text=null / missing text / attachment_ids=null in a message.""" +from __future__ import annotations + +import hashlib +import subprocess +import time +import uuid +from pathlib import Path + +import pytest + +import imagegen +from conftest import attachment_row, compose, s3_size + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +MiB = 1024 * 1024 + + +@pytest.fixture() +def trio(api): + """alice + bob share a chat; eve is an outsider with her own chat.""" + alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") + return alice, bob, eve, api.chat(alice, bob), api.chat(eve) + + +def content(api, actor, attachment_id, expected=200, query=""): + return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"]) + + +# ------------------------------------------------------------------ positive path +def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio): + alice, bob, _, chat, _ = trio + jpg = imagegen.jpeg(321, 123, noise=True) + for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")): + att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json() + assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None + assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] + assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest() + assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin" + meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json() + assert meta == att + got = content(api, bob, att["id"]) + assert got.content == data and got.headers["content-type"] == ctype + assert got.headers["content-length"] == str(len(data)) + assert got.headers["x-content-type-options"] == "nosniff" + assert got.headers["content-disposition"].startswith("attachment;") + assert "location" not in got.headers # bytes through the API, never a redirect to S3 + + +def test_metadata_reveals_no_storage_details(api, trio): + alice, _, _, chat, _ = trio + att = api.upload(alice, chat, SAMPLE).json() + row = attachment_row(att["id"]) + assert row["status"] == "ready" and row["object_key"].startswith("o/") + text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text + assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower() + # The key is random: neither the id nor the filename is part of it. + assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"] + assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket + + +def test_original_is_not_stored_on_api_containers(api, trio): + """The API keeps no file of the original: no recently written file of that size anywhere in + the replica's filesystem (the bytes live only in the bucket).""" + alice, _, _, chat, _ = trio + data = imagegen.png_of_size(300_000) + api.upload(alice, chat, data) + size = len(data) + for cid in compose("ps", "-q", "api").stdout.split(): + out = subprocess.run( + ["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"], + capture_output=True, text=True, timeout=60).stdout.strip() + assert out == "", out + + +# ------------------------------------------------------------------ access control +def test_outsider_and_unknown_get_404(api, trio): + alice, _, eve, chat, _ = trio + att = api.upload(alice, chat, SAMPLE).json() + for suffix in ("", "/content"): + api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + # Upload into a foreign chat: 404 before anything about the body is looked at. + api.upload(eve, chat, SAMPLE, 404) + api.upload(eve, chat, b"garbage", 404, content_type="text/plain") + + +def test_attachment_ids_are_checked_per_chat(api, trio): + alice, bob, eve, chat, eve_chat = trio + other = api.chat(alice) # alice's second chat + att_other = api.upload(alice, other, SAMPLE).json() + att_eve = api.upload(eve, eve_chat, SAMPLE).json() + # Alice is a member of both chats, but a file is bound to its chat. + api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404) + # Eve cannot post into alice's chat even with her own file: membership first. + api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404) + # Any member may reference an attachment of the chat. + att = api.upload(alice, chat, SAMPLE).json() + msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json() + assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"] + + +# ------------------------------------------------------------------ messages with attachments +def test_message_with_attachments_and_history(api, trio): + alice, bob, _, chat, _ = trio + a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2)) + m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json() + m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok + m3 = api.post_message(alice, chat, {"text": "без файлов"}).json() + assert m1["attachment_ids"] == [a2, a1] and m1["text"] == "" + assert m2["text"] == " " and m3["attachment_ids"] == [] + items, _ = api.history(bob, chat, 2) + assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []] + + +def test_idempotent_replay_with_attachments(api, trio): + alice, _, _, chat, _ = trio + a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2)) + key = uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json() + raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode() + assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant + api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body + api.post_message(alice, chat, {"text": "файлы"}, 409, key=key) + # A failed attachment check does not burn the key: the same key works with a valid body later. + key2 = uuid.uuid4().hex + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2) + + +@pytest.mark.parametrize("raw", [ + b'{"attachment_ids":["%(a)s"]}', # text missing + b'{"text":null,"attachment_ids":["%(a)s"]}', # text null + b'{"text":1,"attachment_ids":["%(a)s"]}', + b'{"text":"x","attachment_ids":null}', # attachment_ids null + b'{"text":"x","attachment_ids":"%(a)s"}', + b'{"text":"x","attachment_ids":{}}', + b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates + b'{"text":"x","attachment_ids":["%(a)s",1]}', + b'{"text":"x","attachment_ids":["not-a-uuid"]}', + b'{"text":"","attachment_ids":[]}', # neither text nor files + b'{"text":" "}', + b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}', +], ids=lambda raw: raw.decode()[:40]) +def test_message_body_validation(api, trio, raw): + alice, _, _, chat, _ = trio + a = api.upload(alice, chat, SAMPLE).json()["id"] + api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()}) + + +def test_at_most_ten_attachments(api, trio): + alice, _, _, chat, _ = trio + ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)] + assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10 + api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400) + + +# ------------------------------------------------------------------ limits +def test_size_limit_through_the_load_balancer(api, trio): + alice, bob, _, chat, _ = trio + exact = imagegen.png_exactly(10 * MiB) + assert len(exact) == 10 * MiB + att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB + assert content(api, bob, att["id"]).content == exact + started = time.monotonic() + api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit + assert time.monotonic() - started < 5 + over = exact + bytes(1024) + + def chunks(): # no Content-Length: chunked body + for i in range(0, len(over), 256 * 1024): + yield over[i:i + 256 * 1024] + error = api.upload(alice, chat, chunks(), 413).json() + assert error["error"]["code"] == "payload_too_large" + + +def test_pixel_limit_before_decoding(api, trio): + alice, _, _, chat, _ = trio + for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"), + (imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")): + started = time.monotonic() + assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large" + assert time.monotonic() - started < 2 # refused from the header, nothing decoded + + +def test_exactly_40_megapixels_is_accepted(api, trio): + alice, _, _, chat, _ = trio + data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file + att = api.upload(alice, chat, data).json() + assert att["size_bytes"] == len(data) + + +# ------------------------------------------------------------------ content checks +BROKEN = { + "png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"), + "png: truncated": (SAMPLE[:-20], "image/png"), + "png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"), + "png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"), + "png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"), + "png: bytes after IEND": (SAMPLE + b"", "image/png"), + "jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"), + "jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"), + "mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"), + "mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"), + "mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"), + "mime: html declared png": (b"", "image/png"), + "type: text/plain": (SAMPLE, "text/plain"), + "type: octet-stream": (SAMPLE, "application/octet-stream"), + "type: multipart": (SAMPLE, "multipart/form-data; boundary=x"), + "type: missing": (SAMPLE, None), +} + + +@pytest.mark.parametrize("name", list(BROKEN)) +def test_bad_content_is_415(api, trio, name): + alice, _, _, chat, _ = trio + data, ctype = BROKEN[name] + error = api.upload(alice, chat, data, 415, content_type=ctype).json() + assert error["error"]["code"] == "unsupported_media_type" + + +def test_empty_body_is_400(api, trio): + alice, _, _, chat, _ = trio + api.upload(alice, chat, b"", 400) + + +# ------------------------------------------------------------------ filename +@pytest.mark.parametrize("query", [ + "", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png", + "filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe", + "filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256, +], ids=lambda q: q[:40] or "missing") +def test_bad_filename_is_400(api, trio, query): + alice, _, _, chat, _ = trio + api.upload(alice, chat, SAMPLE, 400, query=query) + + +def test_unicode_filename_is_kept_and_safely_exposed(api, trio): + alice, _, _, chat, _ = trio + name = 'отчёт "Q3"; ok.png' + att = api.upload(alice, chat, SAMPLE, filename=name).json() + assert att["filename"] == name + disposition = content(api, alice, att["id"]).headers["content-disposition"] + assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback + assert "filename*=UTF-8''" in disposition + + +# ------------------------------------------------------------------ variants +def test_variant_parameter(api, trio): + alice, _, eve, chat, _ = trio + att = api.upload(alice, chat, SAMPLE).json() + assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE + content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4 + content(api, alice, att["id"], 400, query="?variant=huge") + content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 00000000..01f3d6ed --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,334 @@ +"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4). + +Each test that stops, kills, rescales or isolates a replica restores the stand in `finally` +and waits until HAProxy reports every replica UP again (conftest.wait_stand). +""" +from __future__ import annotations + +import contextlib +import json +import subprocess +import threading +import time +import uuid + +import httpx2 as httpx +import pytest + +from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \ + running_replicas, wait_healthy, wait_stand + +pytestmark = pytest.mark.stack + + +def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]: + return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)] + + +# ------------------------------------------------------------------ level 3: replicas, no affinity +def test_one_session_is_served_by_every_real_replica(api): + actor = api.user("rr") + seen = instances_seen(api, actor["token"]) + running = {r["hostname"] for r in running_replicas()} + assert len(running) == REPLICAS >= 2 + assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers" + # Round robin on ONE keep-alive client connection: consecutive requests switch replicas, + # so there is no sticky session hiding split state. + switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b) + assert switches >= len(seen) // 2, seen + assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running + + +def test_write_on_one_replica_is_read_on_the_other(api): + alice, bob = api.user("rw"), api.user("rw") + chat = api.chat(alice, bob) + for i in range(10): + sent = api.post_message(alice, chat, {"text": f"m{i}"}) + read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]) + assert read.json()["items"][-1] == sent.json() + logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + for _ in range(6): # the revocation is visible on every replica at once (shared Valkey) + r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + assert r.headers["x-instance-id"] + assert logout.headers["x-instance-id"] + + +# ------------------------------------------------------------------ readiness drives the LB +def hba_file() -> str: + return psql("SHOW hba_file") + + +@contextlib.contextmanager +def isolated_from_postgres(replica: dict): + """Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its + connections), while it keeps running and answering /health/live. Always undone.""" + ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network + path = hba_file() + marker = "lab3-test-isolate" + compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}") + try: + psql("SELECT pg_reload_conf()") + psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'") + yield replica + finally: + compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}") + psql("SELECT pg_reload_conf()") + lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed + wait_stand() + + +@contextlib.contextmanager +def disconnected_from_data_network(replica: dict): + """`docker network disconnect _data `: the replica keeps its edge link to the + LB but loses PostgreSQL AND Valkey. Always reconnected.""" + network = next(n for n in json.loads(subprocess.run( + ["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]], + capture_output=True, text=True, check=True).stdout) if n.endswith("_data")) + subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True) + try: + yield + finally: + subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True) + lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout) + wait_stand() + + +@pytest.mark.parametrize("how", ["pg_hba", "network"]) +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how): + """The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data + network (docker network disconnect) but keeps running. /health/live 200, /health/ready + 503 -> HAProxy must take it out; no business request may reach it.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + actor = api.user("ready") + isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network + with isolate(replica): + started = time.monotonic() + while replica["hostname"] in lbctl.up_instances(): + assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s" + time.sleep(0.1) + detected = time.monotonic() - started + for _ in range(3): # the process stays responsive (event loop never blocked) ... + started_probe = time.monotonic() + assert probe_inside(replica, "/health/live") == 200 + assert probe_inside(replica, "/health/ready") == 503 # ... but not ready + assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable" + seen = instances_seen(api, actor["token"], 40) + assert replica["hostname"] not in seen and len(seen) == 40 + for i in range(20): # writes keep working, only through the ready replicas + assert api.register(f"r{i}")["id"] + print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s") + + +# ------------------------------------------------------------------ stopping replicas under traffic +class Traffic: + """Background writers/readers through the LB; uncertain POSTs are retried with the SAME key.""" + + def __init__(self, api: Api, workers: int = 4): + self.alice, self.bob = api.user("load"), api.user("load") + self.chat = api.chat(self.alice, self.bob) + self.workers = workers + self.stop = threading.Event() + self.lock = threading.Lock() + self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance) + self.confirmed: list[dict] = [] + self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)] + + def record(self, kind: str, status: int, instance: str | None) -> None: + with self.lock: + self.events.append((time.monotonic(), kind, status, instance)) + + def run(self, index: int) -> None: + client = httpx.Client(base_url=BASE_URL, timeout=3) + auth = {"Authorization": "Bearer " + self.alice["token"]} + n = 0 + key, text = uuid.uuid4().hex, f"w{index}-0" + while not self.stop.is_set(): + try: + r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text}, + headers={**auth, "Idempotency-Key": key}) + self.record("POST", r.status_code, r.headers.get("x-instance-id")) + if r.status_code == 201: + with self.lock: + self.confirmed.append(r.json()) + n += 1 + key, text = uuid.uuid4().hex, f"w{index}-{n}" + # any other outcome: keep key and text, retry the same request + except httpx.HTTPError: + self.record("POST", 0, None) + try: + r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth) + self.record("GET", r.status_code, r.headers.get("x-instance-id")) + except httpx.HTTPError: + self.record("GET", 0, None) + time.sleep(0.02) + client.close() + + def __enter__(self) -> "Traffic": + for t in self.threads: + t.start() + time.sleep(1.0) + return self + + def __exit__(self, *exc) -> None: + self.stop.set() + for t in self.threads: + t.join(timeout=10) + + def failures(self, since: float = 0.0) -> list[tuple]: + return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)] + + def check_history(self, api: Api) -> None: + history = api.history(self.bob, self.chat, 100)[0] + ids, texts = [m["id"] for m in history], [m["text"] for m in history] + assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry" + assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost" + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_graceful_stop_of_each_replica_loses_no_request(api, position): + """SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + with Traffic(api) as traffic: + started = time.monotonic() + with replica_stopped(replica, "stop"): + stopped_after = time.monotonic() - started + time.sleep(3) + after = [e for e in traffic.events if e[0] > time.monotonic() - 2] + assert after and replica["hostname"] not in {e[3] for e in after} + time.sleep(1) + state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, + check=True).stdout)[0]["State"] + assert traffic.failures() == [], traffic.failures()[:10] + assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)" + assert state["Status"] == "running" # restored by replica_stopped + traffic.check_history(api) + print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed") + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position): + """SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within + 10 s, the killed replica gets nothing after detection, retries do not duplicate.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + with Traffic(api) as traffic: + killed_at = time.monotonic() + with replica_stopped(replica, "kill"): + time.sleep(4) + failures = traffic.failures(killed_at) + last_failure = max((e[0] for e in failures), default=killed_at) - killed_at + post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201] + assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s" + late = [e for e in traffic.events if e[0] > killed_at + 3.5] + assert late and replica["hostname"] not in {e[3] for e in late} + assert last_failure <= 10 + traffic.check_history(api) + print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s") + + +def test_graceful_shutdown_order_in_logs(api): + """Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[-1] + since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1)) + with replica_stopped(replica, "stop"): + logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True, + text=True, encoding="utf-8").stdout + lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout + state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, + check=True).stdout)[0]["State"] + events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] + names = [e["event"] for e in events] + assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped") + t = {e["event"]: e["ts"] for e in events} + assert t["drain_started"] < t["shutdown_requested"] + assert state["ExitCode"] == 0 and not state["OOMKilled"] + assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs) + + +# ------------------------------------------------------------------ discovery: restart and scaling +def test_full_restart_lb_rediscovers_all_replicas(api): + actor = api.user("restart") + before = {r["hostname"]: r["started_at"] for r in running_replicas()} + run_hook("restart.sh") + wait_healthy("postgres", "valkey", "api") + wait_stand() + after = {r["hostname"]: r["started_at"] for r in running_replicas()} + assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted" + assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed + + +def test_scale_up_and_down_is_picked_up_without_errors(api): + with Traffic(api, workers=2) as traffic: + try: + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api", + timeout=180) + lbctl.wait_up(REPLICAS + 1, timeout=30) + actor = api.user("scale") + assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1 + finally: + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api", + timeout=180) + wait_stand() + time.sleep(1) + assert traffic.failures() == [], traffic.failures()[:10] + traffic.check_history(api) + + +# ------------------------------------------------------------------ level 4: deployment policy +def test_compose_policy_limits_ports_secrets_images(): + config = json.loads(compose("config", "--format", "json").stdout) + services = config["services"] + # The reconciler (profile "tools") is started on demand and not part of the running stand. + assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"} + for name, svc in services.items(): + limits = svc.get("deploy", {}).get("resources", {}).get("limits", {}) + assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits" + assert "container_name" not in svc, f"{name}: container_name prevents scaling" + if name in ("api", "migrate"): + assert "build" in svc # built from this repository's source + elif name == "s3-init": + assert svc["image"] == services["api"]["image"] # the same locally built image + else: + assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest" + assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports") + published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")} + assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"]) + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()} + s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key'] + assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys, + "migrate": ["database_url"], "s3-init": s3_keys, "lb": [], + # the API: its own S3 key pair only (no admin, no reconciler keys) + "api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]} + volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] + for name, svc in services.items()} + assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [], + "s3-init": [], "api": [], "lb": []} + networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()} + assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"], + "s3-init": ["storage"], "api": ["data", "edge", "storage"], + "lb": ["edge"]}, networks # the LB cannot reach the stores at all + assert config["networks"]["data"].get("internal") is True + assert config["networks"]["storage"].get("internal") is True + + +def test_running_containers_have_limits_and_no_bypass_ports(): + ids = compose("ps", "-q").stdout.split() + for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True, + check=True).stdout): + name = item["Name"] + assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name + bindings = item["HostConfig"]["PortBindings"] or {} + if "-lb-" in name: + assert all(b["HostIp"] in ("127.0.0.1", "::1") for v in bindings.values() for b in v) # loopback only + else: + assert not bindings, f"{name} publishes ports in bypass of the LB" + + +def test_migrations_ran_once_by_the_migrate_service(): + applied = psql("SELECT version FROM schema_migrations ORDER BY version").split() + assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"] + migrate = compose("ps", "-a", "--format", "json", "migrate").stdout + rows = [json.loads(line) for line in migrate.splitlines() if line.strip()] + assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows) + api_logs = compose("logs", "--no-log-prefix", "api").stdout + assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 00000000..9e375a94 --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,111 @@ +"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4). + +The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between +the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API +raises 503 at that exact point, as if the process or the network failed there): + +* after_s3_put object written, row still pending, client got 503 -> the janitor removes + the object and the row after PENDING_GRACE_SECONDS (here 3 s); +* after_ready_commit row ready and object written, but the answer is lost (client got 503) + -> nothing may be deleted: the attachment stays intact and downloadable. + +Plus: an upload interrupted by the client and a stalled body leave nothing behind.""" +from __future__ import annotations + +import socket +import time +from pathlib import Path + +import pytest + +import imagegen +from conftest import api_with_env, attachment_row, psql, s3_size + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]: + out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'") + return [tuple(line.split()) for line in out.splitlines() if line] + + +def wait_for(condition, timeout: float, what: str) -> None: + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, what + time.sleep(0.5) + + +def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes: + """Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the + socket (client gone) or keep it open and read the answer after `wait_answer` seconds.""" + sock = socket.create_connection(("127.0.0.1", 8080), timeout=30) + head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n" + f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n") + sock.sendall(head.encode() + body) + if wait_answer is None: + sock.close() + return b"" + time.sleep(wait_answer) + answer = sock.recv(65536) + sock.close() + return answer + + +def test_interrupted_and_stalled_uploads_leave_nothing(api): + alice = api.user("alice") + chat = api.chat(alice) + data = imagegen.png_of_size(2_000_000) + raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects + with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"): + answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls + assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200] + assert b"x-instance-id" in answer.lower() + time.sleep(1) + assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write + + +def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api): + alice = api.user("alice") + chat = api.chat(alice) + with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): + assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable" + [(attachment_id, status, key)] = rows_of_chat(chat["id"]) + assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists + api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible + api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable + wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent") + assert s3_size(key) is None # object removed first + # Normal configuration again: an upload in the same chat works and is not touched. + att = api.upload(alice, chat, SAMPLE).json() + assert attachment_row(att["id"])["status"] == "ready" + + +def test_lost_answer_after_commit_never_deletes_the_object(api): + """The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API + 'compensated' by deleting the object -> ready row without bytes. Here nothing compensates + in the request path, and the janitor only touches 'pending' rows.""" + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): + api.upload(alice, chat, SAMPLE, 503) + [(attachment_id, status, key)] = rows_of_chat(chat["id"]) + assert status == "ready" + time.sleep(6) # several janitor rounds + assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE) + got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"]) + assert got.content == SAMPLE + + +def test_fresh_pending_upload_is_not_touched_by_janitor(api): + """An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed.""" + alice = api.user("alice") + chat = api.chat(alice) + with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"): + api.upload(alice, chat, SAMPLE, 503) + time.sleep(4) + [(_, status, key)] = rows_of_chat(chat["id"]) + assert status == "pending" and s3_size(key) == len(SAMPLE) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..544d3781 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,117 @@ +"""Database-level guarantees, checked over HTTP and verified directly in PostgreSQL / Valkey.""" +from __future__ import annotations + +import threading +import time +import uuid + +import pytest + +from conftest import PASSWORD, Api, compose, psql + +pytestmark = pytest.mark.stack + + +def run_concurrently(workers: int, target) -> list: + barrier = threading.Barrier(workers) + results: list = [None] * workers + + def body(index: int) -> None: + local = Api() # one HTTP client (connection) per worker + barrier.wait() # all workers fire at the same moment + try: + results[index] = target(local, index) + except BaseException as exc: # noqa: BLE001 - reported by the test + results[index] = exc + + threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + return results + + +def test_registration_race_exactly_one_201_and_seven_409(): + username = "race_" + uuid.uuid4().hex[:10] + + def register(client: Api, _index: int) -> int: + body = {"username": username, "display_name": "Гонка", "password": PASSWORD} + response = client.client.post("/api/v1/users", json=body) + return response.status_code + + statuses = run_concurrently(8, register) + # Real HTTP codes are compared: a timeout, 500 or exception fails the test instead of counting as 409. + assert sorted(statuses, key=str) == [201] + [409] * 7, statuses + assert psql(f"SELECT count(*) FROM users WHERE username = '{username}'") == "1" + + +def _ins_counter() -> int: + return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + +def test_create_chat_rolls_back_after_the_chat_row_was_inserted(api): + """The chat INSERT runs first; the unknown member then breaks the membership INSERT (FK). + + Proof that the first INSERT really happened and was rolled back: PostgreSQL counts + attempted inserts in pg_stat_user_tables.n_tup_ins even for aborted transactions, + while the table contains no such chat and no new membership rows. + """ + alice = api.user("rollback") + title = "rollback-" + uuid.uuid4().hex + members_before = int(psql("SELECT count(*) FROM chat_members")) + attempts_before = _ins_counter() + error = api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}).json() + assert "несуществующих" in error["error"]["message"] + deadline = time.monotonic() + 20 # statistics are flushed by the API backend asynchronously + while _ins_counter() == attempts_before and time.monotonic() < deadline: + time.sleep(0.5) + assert _ins_counter() > attempts_before, "the chat INSERT was never attempted" + assert psql(f"SELECT count(*) FROM chats WHERE title = convert_to('{title}', 'UTF8')") == "0" + assert int(psql("SELECT count(*) FROM chat_members")) == members_before + + +def test_migrations_rerun_is_a_noop(): + users_before = psql("SELECT count(*) FROM users") + result = compose("run", "--rm", "--no-deps", "migrate") + assert "0001_init: already applied, skipped" in result.stdout + assert "0002_idempotency_keys: already applied, skipped" in result.stdout + assert "0003_post_message_function: already applied, skipped" in result.stdout + assert "0004_attachments: already applied, skipped" in result.stdout + assert psql("SELECT count(*) FROM schema_migrations") == "4" + assert psql("SELECT count(*) FROM users") == users_before + + +def test_100_messages_from_10_clients_keep_order_and_cursor_walk(api): + clients = [api.user(f"w{i}") for i in range(10)] + chat = api.chat(clients[0], *clients[1:]) + + def writer(client: Api, index: int) -> list[dict]: + return [client.send(clients[index], chat, f"c{index}-m{j}") for j in range(10)] + + results = run_concurrently(10, writer) + assert all(isinstance(r, list) for r in results), results + confirmed = [m for batch in results for m in batch] + history, pages = api.history(clients[5], chat, 7) + ids = [m["id"] for m in history] + assert len(confirmed) == 100 and len(history) == 100 and len(set(ids)) == 100 and pages == 15 + assert set(ids) == {m["id"] for m in confirmed} + keys = [(m["created_at"], uuid.UUID(m["id"]).int) for m in history] + assert all(a < b for a, b in zip(keys, keys[1:])), "history is not strictly ordered" + stamps = [m["created_at"] for m in history] + assert len(set(stamps)) == 100, "per-chat clock must give unique timestamps" + for i, client in enumerate(clients): + own = [m["text"] for m in history if m["sender_id"] == client["id"]] + assert own == [f"c{i}-m{j}" for j in range(10)] + + +def test_secrets_are_not_stored_in_plain_text(api): + actor = api.user("store") + stored = psql(f"SELECT password_hash FROM users WHERE id = '{actor['id']}'") + assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and PASSWORD not in stored + token_hex = actor["token"].encode().hex() + keys = compose("exec", "-T", "valkey", "sh", "-c", + 'REDISCLI_AUTH="$(cat /run/secrets/valkey_password)" valkey-cli --scan --pattern "session:*"').stdout.split() + assert keys and all(len(k) == len("session:") + 64 for k in keys) + assert not any(actor["token"] in k or token_hex in k for k in keys) diff --git a/student-tests/test_explain.py b/student-tests/test_explain.py new file mode 100644 index 00000000..a2503a5f --- /dev/null +++ b/student-tests/test_explain.py @@ -0,0 +1,35 @@ +"""The history index really bounds the work of a late page (>= 10 000 messages, generic plan).""" +from __future__ import annotations + +import re +import sys + +import pytest + +from conftest import ROOT + +pytestmark = pytest.mark.stack + +sys.path.insert(0, str(ROOT / "scripts")) +import explain_history # noqa: E402 + + +def section(text: str, number: int) -> str: + parts = re.split(r"^=== ", text, flags=re.M) + return next(p for p in parts if p.startswith(f"{number}.")) + + +def buffers(plan: str) -> int: + return int(re.search(r"Buffers: shared hit=(\d+)", plan).group(1)) + + +def test_late_page_uses_index_range_not_filter(): + text = explain_history.run() + assert "20000" in text # the target chat holds 20 000 messages + app_late = section(text, 2) + assert "Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))" in app_late + assert "Rows Removed by Filter" not in app_late and "Sort" not in app_late + rejected = section(text, 3) + assert "Rows Removed by Filter: 19950" in rejected + # The app query touches a handful of pages; the rejected variants touch the whole prefix. + assert buffers(app_late) <= 10 < buffers(rejected) and buffers(app_late) <= 10 < buffers(section(text, 4)) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..899d29b8 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,111 @@ +"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards. + +Every test stops/pauses a real container and restores it in `finally` (service_down). +Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5. + +Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer +503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same +contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401 +for a valid token. Liveness/readiness of the processes are checked inside each container. +""" +from __future__ import annotations + +import subprocess +import time + +import pytest + +from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down + +pytestmark = pytest.mark.stack + +DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead +CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead + + +def timed(api, *args, **kwargs): + started = time.monotonic() + response = api.call(*args, lb_error_ok=True, **kwargs) + return response, time.monotonic() - started + + +def processes_alive_but_not_ready() -> None: + for replica in running_replicas(): + assert probe_inside(replica, "/health/live") == 200 + assert probe_inside(replica, "/health/ready") == 503 + + +@pytest.fixture +def prepared(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + message = api.send(alice, chat, "до отказа") + return alice, bob, chat, message + + +def check_intact(api, alice, bob, chat, message): + """After recovery the SAME tokens work and confirmed data is unchanged.""" + assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"] + items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"] + assert items == [message] + + +@pytest.mark.parametrize("how", ["stop", "pause"]) +def test_valkey_failure(api, prepared, how): + alice, bob, chat, message = prepared + with service_down("valkey", how): + response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"]) + assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took + api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access + api.call("GET", "/health/ready", 503, lb_error_ok=True) + processes_alive_but_not_ready() + check_intact(api, alice, bob, chat, message) + + +@pytest.mark.parametrize("how", ["stop", "pause"]) +def test_postgres_failure(api, prepared, how): + alice, bob, chat, message = prepared + with service_down("postgres", how): + for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + response, took = timed(api, "GET", path, 503, token=alice["token"]) + assert response.json()["error"]["code"] == "unavailable", path + assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause) + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access + api.call("GET", "/health/ready", 503, lb_error_ok=True) + processes_alive_but_not_ready() + check_intact(api, alice, bob, chat, message) + + +def test_locked_table_gives_bounded_503(api, prepared): + """A statement stuck behind a lock is stopped by lock_timeout, not after minutes.""" + alice, bob, chat, message = prepared + locker = subprocess.Popen( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", + "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + time.sleep(1.5) # let the lock be taken + response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) + assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took + assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready" + finally: + locker.wait(timeout=30) + check_intact(api, alice, bob, chat, message) + + +def test_logout_during_valkey_outage_is_never_lost(api, prepared): + """Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten': + during the outage logout answers 503 (nothing revoked, the client knows it failed); + after recovery logout works, and the revocation survives a Valkey and API restart (AOF).""" + alice, bob, chat, message = prepared + with service_down("valkey", "stop"): + api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True) + api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + restart_services("valkey", "api") + for _ in range(10): # repeated checks: the revoked token must never come back + api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"}) + api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..a1e2bccc --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,281 @@ +"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer. + +Strict assertions: exact HTTP codes for every request, identical bodies for replays, +the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica +answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always +undone in `finally`. +""" +from __future__ import annotations + +import subprocess +import threading +import time +import uuid + +import httpx2 as httpx +import pytest + +from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas + +pytestmark = pytest.mark.stack + + +def messages_in_db(chat: dict) -> int: + return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'")) + + +def key_rows(actor: dict, key: str) -> int: + return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'")) + + +@pytest.fixture +def pair(api): + alice, bob = api.user("alice"), api.user("bob") + return alice, bob, api.chat(alice, bob) + + +def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair): + alice, bob, chat = pair + key = "replay-" + uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "один раз"}, key=key) + served = {first.headers["x-instance-id"]} + for _ in range(20): + again = api.post_message(alice, chat, {"text": "один раз"}, key=key) + assert again.status_code == 201 and again.json() == first.json() + served.add(again.headers["x-instance-id"]) + assert len(served) >= 2, "replays must be answered by different replicas, not a local cache" + assert api.history(bob, chat, 100)[0] == [first.json()] + assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair): + alice, bob, chat = pair + key = uuid.uuid4().hex + original = api.send(alice, chat, "оригинал", key=key) + for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}): + assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict" + assert api.send(alice, chat, "оригинал", key=key) == original + assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1 + + +def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair): + alice, _, chat = pair + key = uuid.uuid4().hex + first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json() + for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}', + '{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'): + assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first + assert messages_in_db(chat) == 1 + + +@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"]) +def test_missing_or_invalid_key_is_400(api, pair, value): + alice, _, chat = pair + raw_value = value.encode("utf-8") + error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json() + assert error["error"]["code"] == "invalid_request" + assert messages_in_db(chat) == 0 + + +def test_repeated_key_header_is_400(pair): + alice, _, chat = pair + response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10, + headers=[("Authorization", "Bearer " + alice["token"]), + ("Idempotency-Key", "one"), ("Idempotency-Key", "two")]) + assert response.status_code == 400 and response.headers["x-instance-id"] + assert messages_in_db(chat) == 0 + + +def test_longest_key_and_every_allowed_character(api, pair): + alice, _, chat = pair + for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex): + first = api.send(alice, chat, key[:20], key=key) + assert api.send(alice, chat, key[:20], key=key) == first + assert messages_in_db(chat) == 2 + + +def test_scope_is_user_plus_chat_plus_key(api, pair): + alice, bob, chat = pair + other = api.chat(alice, bob, title="второй") + key = "shared-key" + a1 = api.send(alice, chat, "текст", key=key) + b1 = api.send(bob, chat, "текст", key=key) # another user: independent + a2 = api.send(alice, other, "текст", key=key) # another chat: independent + assert len({a1["id"], b1["id"], a2["id"]}) == 3 + assert messages_in_db(chat) == 2 and messages_in_db(other) == 1 + + +def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair): + alice, bob, chat = pair + eve = api.user("eve") + key = uuid.uuid4().hex + api.send(alice, chat, "секрет", key=key) + for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message + assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found" + api.post_message(eve, chat, {"text": "другое"}, 404, key=key) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again + assert messages_in_db(chat) == 1 + + +def fire_together(workers: int, request) -> list: + barrier = threading.Barrier(workers) + results: list = [None] * workers + + def body(index: int) -> None: + client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them + barrier.wait() + try: + results[index] = request(client, index) + except BaseException as exc: # noqa: BLE001 - asserted by the test + results[index] = exc + finally: + client.close() + + threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + return results + + +def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair): + alice, bob, chat = pair + key = "race-" + uuid.uuid4().hex + + def post(client: httpx.Client, _index: int): + r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"}, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + return r.status_code, r.headers.get("x-instance-id"), r.json() + + for _ in range(3): # three rounds: the first creates, later rounds replay concurrently + results = fire_together(24, post) + assert all(isinstance(r, tuple) for r in results), results + assert [r[0] for r in results] == [201] * 24, [r[0] for r in results] + assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message" + assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas" + assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_concurrent_same_key_different_bodies_one_wins(pair): + alice, _, chat = pair + key = "conflict-" + uuid.uuid4().hex + + def post(client: httpx.Client, index: int): + r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"}, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + return r.status_code, r.json() + + results = fire_together(20, post) + winners = [body for status, body in results if status == 201] + losers = [body for status, body in results if status == 409] + assert len(winners) + len(losers) == 20, results + assert winners and len({str(b) for b in winners}) == 1 + assert all(b["error"]["code"] == "conflict" for b in losers) + assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"} + assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"]) + assert messages_in_db(chat) == 1 + + +def inserts_into_keys() -> int: + return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'")) + + +def test_failure_after_the_key_insert_rolls_everything_back(api, pair): + """A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same + transaction. Expected: 500 for the client, no key row and no message left behind + (n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds.""" + alice, bob, chat = pair + marker = "boom-" + uuid.uuid4().hex + key = uuid.uuid4().hex + psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ " + f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; " + "RETURN NEW; END $$; " + "CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()") + try: + before = inserts_into_keys() + error = api.post_message(alice, chat, {"text": marker}, 500, key=key) + assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text + assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 + deadline = time.monotonic() + 20 # table statistics are flushed asynchronously + while inserts_into_keys() == before and time.monotonic() < deadline: + time.sleep(0.5) + assert inserts_into_keys() > before, "the key INSERT never ran before the failure" + finally: + psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()") + retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded + assert api.send(alice, chat, marker, key=key) == retried + assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1 + + +def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen: + sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; " + f"SELECT pg_sleep({seconds}); COMMIT;") + return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", + "messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def wait_lock_taken(chat: dict) -> None: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' " + "AND pid <> pg_backend_pid()") != "0": + return + time.sleep(0.2) + raise AssertionError("the row lock was not taken") + + +def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair): + """Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row + lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one.""" + alice, bob, chat = pair + key = uuid.uuid4().hex + locker = lock_chat_row(chat, 1.0) + try: + wait_lock_taken(chat) + with pytest.raises(httpx.TimeoutException): + httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + finally: + locker.wait(timeout=30) + deadline = time.monotonic() + 10 + while messages_in_db(chat) == 0 and time.monotonic() < deadline: + time.sleep(0.2) + stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'") + assert stored, "the server-side transaction was expected to commit after the lock was released" + retried = api.send(alice, chat, "неопределённо", key=key) + assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair): + """The row lock outlives lock_timeout: the API answers 503 within its deadline and stores + nothing; after the lock is gone the same key creates the message exactly once.""" + alice, bob, chat = pair + key = uuid.uuid4().hex + locker = lock_chat_row(chat, 4.0) + try: + wait_lock_taken(chat) + started = time.monotonic() + api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key) + assert time.monotonic() - started < 3.6 + assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 + finally: + locker.wait(timeout=30) + created = api.send(alice, chat, "после блокировки", key=key) + assert api.send(alice, chat, "после блокировки", key=key) == created + assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1 + + +def test_replay_after_the_serving_replica_is_stopped(api, pair): + alice, bob, chat = pair + key = uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "до остановки"}, key=key) + served_by = first.headers["x-instance-id"] + replica = next(r for r in running_replicas() if r["hostname"] == served_by) + with replica_stopped(replica): + for _ in range(5): + again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key) + assert again.json() == first.json() and again.headers["x-instance-id"] != served_by + assert messages_in_db(chat) == 1 diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 00000000..d32a7508 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,217 @@ +"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile). + +Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key +pairs from inside the storage network, PostgreSQL with psql): + + S3 side: orphan object (no row), object deleted under a ready row, object overwritten with + other bytes (size / same-size checksum), object of a quarantined row restored; + DB side: stale intent with and without object, quarantine mark on a healthy row; + fresh: orphan and intent younger than --grace (an upload in progress). + +The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings; +fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine); +the second run finds nothing to do; the safety valve and store outages stop the run without +damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast.""" +from __future__ import annotations + +import concurrent.futures +import hashlib +import time +import uuid +from pathlib import Path + +import pytest + +import imagegen +from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size, + service_down) + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +GRACE = ["--grace", "8", "--allow-small-grace"] +# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot +# (each docker exec costs 1-3 s on Windows), hence a longer grace there. +LONG_GRACE = ["--grace", "30", "--allow-small-grace"] + + +def new_key() -> str: + return "o/" + uuid.uuid4().hex + + +def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str: + """A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards).""" + attachment_id = str(uuid.uuid4()) + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " + "object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, " + "decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), " + "clock_timestamp() - make_interval(secs => %d))" + % (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds, + age_seconds)) + return attachment_id + + +def snapshot() -> tuple[str, frozenset]: + """Fingerprint of everything the reconciler could change: every attachment row + the key list.""" + rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || " + "coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments") + return rows, frozenset(s3_keys()) + + +def content_status(api, actor, attachment_id) -> int: + return api.client.get(f"/api/v1/attachments/{attachment_id}/content", + headers={"Authorization": "Bearer " + actor["token"]}).status_code + + +def test_detect_and_safely_fix_faults_on_both_sides(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")} + data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content + for name, a in up.items()} + key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()} + api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]}) + + # ---- inject (old) + orphan_old = new_key() + s3_put(orphan_old, SAMPLE) + stale_with, stale_without = new_key(), new_key() + s3_put(stale_with, SAMPLE) + stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s + stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600) + s3_delete(key["C"]) # ready + used, bytes lost + s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size + psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' " + f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine + time.sleep(32) # everything above is now older than --grace + # ---- inject (fresh: an upload in progress) + fresh_orphan, fresh_intent = new_key(), new_key() + s3_put(fresh_orphan, SAMPLE) + s3_put(fresh_intent, SAMPLE) + fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0) + + # ---- dry-run: report everything, change nothing + before = snapshot() + dry = reconcile(*LONG_GRACE) + assert snapshot() == before, "dry-run modified a store" + f = findings_by_key(dry) + expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent", + stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object", + key["D"]: "size_mismatch", key["E"]: "restored_object"} + assert {k: f[k]["kind"] for k in expected} == expected + assert key["A"] not in f and key["B"] not in f + assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run" + assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected) + + # ---- apply + applied = findings_by_key(reconcile("--apply", *LONG_GRACE)) + for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]): + assert applied[k]["result"] == "applied", (k, applied[k]) + keys = s3_keys() + assert orphan_old not in keys and stale_with not in keys + assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None + # fresh: untouched + assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending" + # used, lost bytes: quarantined, NOT deleted; the message still references it + row_c = attachment_row(up["C"]["id"]) + assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object" + items, _ = api.history(bob, chat, 100) + assert any(up["C"]["id"] in m["attachment_ids"] for m in items) + assert content_status(api, bob, up["C"]["id"]) == 404 + api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file + assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch" + assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted + for name in "ABE": + got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"]) + assert got.content == data[name] + + # ---- second run: idempotent, nothing left to do for these keys + again = findings_by_key(reconcile("--apply", *LONG_GRACE)) + for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]): + assert k not in again, (k, again.get(k)) + + # ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again + s3_put(key["C"], data["C"]) + restored = findings_by_key(reconcile("--apply", *LONG_GRACE)) + assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied" + assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"] + # tidy up the fresh items of this test once they are old enough + time.sleep(31) + reconcile("--apply", *LONG_GRACE) + assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None + + +def test_checksum_mode_finds_same_size_corruption(api): + alice = api.user("alice") + chat = api.chat(alice) + att = api.upload(alice, chat, SAMPLE).json() + k = attachment_row(att["id"])["object_key"] + s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes + time.sleep(9) + assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it + report = reconcile("--verify-checksums", *GRACE) + assert findings_by_key(report)[k]["kind"] == "checksum_mismatch" + assert report["cost"]["s3_get_bytes"] >= len(SAMPLE) + assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied" + assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch" + # A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays. + assert k not in findings_by_key(reconcile("--apply", *GRACE)) + assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch" + # Restoring the original and verifying lifts it. + s3_put(k, SAMPLE) + lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k] + assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied") + assert attachment_row(att["id"])["lost_reason"] is None + + +def test_safety_valve_refuses_mass_deletion(): + """If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database - + --apply must stop before deleting anything.""" + orphan = new_key() + s3_put(orphan, SAMPLE) + time.sleep(9) + report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3) + assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0 + assert orphan in s3_keys() + reconcile("--apply", *GRACE) + assert orphan not in s3_keys() + + +@pytest.mark.parametrize("service", ["s3", "postgres"]) +def test_store_outage_stops_the_run_without_damage(api, service): + with service_down(service, "stop"): + report = reconcile(*GRACE, expect_code=1) + assert report["error"].startswith("store_unavailable") + assert reconcile(*GRACE)["error"] is None + + +def test_live_uploads_are_never_hurt_by_concurrent_apply(api): + """Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays + ready and downloadable byte for byte, every message keeps its attachments.""" + alice = api.user("alice") + chat = api.chat(alice) + payloads = [imagegen.png(16 + i, 16) for i in range(40)] + done: list[tuple[dict, bytes]] = [] + + def worker(chunk: list[bytes]) -> None: + for p in chunk: + att = api.upload(alice, chat, p).json() + api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}) + done.append((att, p)) + + with concurrent.futures.ThreadPoolExecutor(4) as pool: + futures = [pool.submit(worker, payloads[i::4]) for i in range(4)] + runs = 0 + while not all(x.done() for x in futures): + report = reconcile("--apply", *GRACE) + assert report["error"] is None + runs += 1 + for x in futures: + x.result() + assert runs >= 1 and len(done) == len(payloads) + for att, p in done: + row = attachment_row(att["id"]) + assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p) + for att, p in done[::8]: + assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..40d793bb --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,66 @@ +"""Session lifetime: expiry with a short SESSION_TTL_SECONDS, no extension by restarts, +and the alternative SESSION_STORE=postgres. The api container is recreated with the +override and ALWAYS recreated back with the normal configuration (api_with_env).""" +from __future__ import annotations + +import datetime as dt +import time + +import pytest + +from conftest import api_env, api_with_env, restart_services, service_down + +pytestmark = pytest.mark.stack + + +def seconds_left(expires_at: str) -> float: + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def login_session(api, actor) -> dict: + from conftest import basic + return api.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json() + + +@pytest.mark.parametrize("store", ["valkey", "postgres"]) +def test_short_ttl_expires(api, store): + with api_with_env(SESSION_TTL_SECONDS="2", SESSION_STORE=store): + assert api_env("SESSION_TTL_SECONDS") == "2" # the contract variable reaches the container + actor = api.register("ttl") + session = login_session(api, actor) + assert 1.0 < seconds_left(session["expires_at"]) <= 2.1 + api.call("GET", "/api/v1/users/me", 200, token=session["token"]) + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3) + api.call("GET", "/api/v1/users/me", 401, token=session["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=session["token"]) + + +def test_restart_does_not_extend_ttl(api): + # 30 s: a restart of Valkey + both API replicas (readiness now also waits for S3) takes ~12-15 s. + with api_with_env(SESSION_TTL_SECONDS="30"): + actor = api.register("ttlrestart") + session = login_session(api, actor) + restart_services("valkey", "api") + assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it" + api.call("GET", "/api/v1/users/me", 200, token=session["token"]) + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3) + api.call("GET", "/api/v1/users/me", 401, token=session["token"]) + + +def test_postgres_session_store(api): + """SESSION_STORE=postgres: sessions survive an API restart, logout revokes, and a DB outage + is 503 for a valid token (not 401 and not anonymous access).""" + with api_with_env(SESSION_STORE="postgres"): + assert api_env("SESSION_STORE") == "postgres" + actor = api.user("pgstore") + restart_services("api") + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + with service_down("postgres", "stop"): + assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"], + lb_error_ok=True).json()["error"]["code"] == "unavailable" + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"]) + api.call("GET", "/api/v1/users/me", 401, token=actor["token"]) diff --git a/student-tests/test_storage_failures.py b/student-tests/test_storage_failures.py new file mode 100644 index 00000000..412b210c --- /dev/null +++ b/student-tests/test_storage_failures.py @@ -0,0 +1,107 @@ +"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4). + +Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the +stand is waited for until HAProxy sees all replicas UP again.""" +from __future__ import annotations + +import time +from pathlib import Path + +import pytest + +import imagegen +from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +@pytest.fixture() +def pair(api): + alice, bob = api.user("alice"), api.user("bob") + return alice, bob, api.chat(alice, bob) + + +@pytest.mark.parametrize("how", ["stop", "pause"]) +def test_s3_outage_is_503_and_text_keeps_working(api, pair, how): + """S3 down (stopped: connection refused; paused: no answer at all): + * readiness is 503 on every replica (and through the LB), quickly; + * uploads/downloads are 503 `unavailable` within the S3 deadline, never 500; + * the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core); + * after recovery the earlier file is intact.""" + alice, bob, chat = pair + att = api.upload(alice, chat, SAMPLE).json() + with service_down("s3", how): + deadline = time.monotonic() + 15 + while api.client.get("/health/ready").status_code != 503: + assert time.monotonic() < deadline, "readiness did not turn 503" + time.sleep(0.5) + for replica in running_replicas(): + assert probe_inside(replica, "/health/ready") == 503 + assert probe_inside(replica, "/health/ready/core") == 200 + started = time.monotonic() + error = api.upload(alice, chat, SAMPLE, 503).json() + assert error["error"]["code"] == "unavailable" + assert time.monotonic() - started < 8 + started = time.monotonic() + api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + assert time.monotonic() - started < 8 + api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only + api.send(alice, chat, "текст работает без S3") # degraded mode + got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + assert got.content == SAMPLE + api.upload(alice, chat, SAMPLE) + + +def test_upload_survives_failover_of_the_serving_replica(api, pair): + """Upload -> stop the replica that served it -> another replica downloads the same bytes + (nothing of the file lives in the replica).""" + alice, bob, chat = pair + data = imagegen.png_of_size(500_000) + response = api.upload(alice, chat, data) + att, served_by = response.json(), response.headers["x-instance-id"] + replica = next(r for r in running_replicas() if r["hostname"] == served_by) + with replica_stopped(replica, "kill"): + for _ in range(5): + got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + assert got.headers["x-instance-id"] != served_by and got.content == data + + +def test_bucket_is_private_outside_the_api(api, pair): + """Inside the storage network (where the API is): an UNSIGNED GET of a real object and an + unsigned LIST are refused; the same GET signed with the API key pair works (control). + SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable.""" + alice, _, chat = pair + att = api.upload(alice, chat, SAMPLE).json() + key = attachment_row(att["id"])["object_key"] + probe = ( + "import urllib.request as u, urllib.error as e, socket\n" + "def st(url):\n" + " try: return u.urlopen(url, timeout=3).status\n" + " except e.HTTPError as x: return x.code\n" + f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n" + "for port in (9333, 8888, 8080):\n" + " s = socket.socket(); s.settimeout(2)\n" + " print(port, s.connect_ex(('s3', port)))\n") + out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n") + get_status, list_status = out[0].split() + assert get_status in ("401", "403") and list_status in ("401", "403"), out[0] + for line in out[1:4]: + port, code = line.split() + assert code != "0", f"admin port {port} reachable" + signed = compose("exec", "-T", "api", "python", "-c", + "import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n" + "async def m():\n s=Storage(Settings.from_env()); await s.open()\n" + f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip() + assert signed == str(len(SAMPLE)) + + +def test_s3_restart_keeps_objects(api, pair): + alice, bob, chat = pair + data = imagegen.png_of_size(200_000) + att = api.upload(alice, chat, data).json() + with service_down("s3", "stop"): + pass + assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data diff --git a/student-tests/unit/test_images.py b/student-tests/unit/test_images.py new file mode 100644 index 00000000..94d57f04 --- /dev/null +++ b/student-tests/unit/test_images.py @@ -0,0 +1,148 @@ +"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services. + +The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it +(the same cases run against the stand in test_attachments.py).""" +from __future__ import annotations + +import asyncio +import shutil +import sys +import time +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "student-tests")) + +import imagegen # noqa: E402 +from messenger import images, validation # noqa: E402 +from messenger.errors import InvalidRequest # noqa: E402 + +LIMIT = 40_000_000 +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed") + + +def status(data: bytes) -> int: + """201 if accepted, else the HTTP status of the ImageError (413/415).""" + try: + info = images.sniff(data, LIMIT) + if info is None: + return 415 # header never completed + asyncio.run(images.verify(data, info)) + return 201 + except images.ImageError as exc: + return exc.status + + +def test_public_fixture_is_accepted(): + assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200) + assert status(SAMPLE) == 201 + + +@pytest.mark.parametrize("data", [ + imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"), + imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1), +], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"]) +def test_valid_png_kinds(data): + assert status(data) == 201 + + +def test_interlaced_png_is_accepted(): + # Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0. + from messenger.images import _png_segments + raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1) + for _ in range(size // stride)) + data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \ + + imagegen.chunk(b"IEND", b"") + assert status(data) == 201 + + +BROKEN_PNG = { + "first 33 bytes of the fixture": SAMPLE[:33], + "truncated (no IEND)": SAMPLE[:-12], + "truncated in IDAT": SAMPLE[:len(SAMPLE) // 2], + "flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:], + "garbage IDAT with valid CRC": imagegen.png_corrupt_idat(), + "fewer rows than IHDR": imagegen.png_short_pixels(), + "bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip", + "text": b"just text, definitely not an image", + "gif": b"GIF89a" + bytes(64), + "two bytes of a signature": b"\x89P", +} + + +@pytest.mark.parametrize("name", list(BROKEN_PNG)) +def test_broken_png_is_415(name): + assert status(BROKEN_PNG[name]) == 415, name + + +def test_pixel_limit_is_checked_from_the_header(): + with pytest.raises(images.ImageError) as exc: + images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough + assert exc.value.status == 413 + exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed + assert images.sniff(exact, LIMIT).width == 8000 + with pytest.raises(images.ImageError) as exc: + images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT) + assert exc.value.status == 413 + with pytest.raises(images.ImageError) as exc: + images.sniff(imagegen.jpeg_bomb_header(), LIMIT) + assert exc.value.status == 413 + + +def test_png_verification_is_streaming_and_fast(): + """40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded.""" + big = imagegen.png(8000, 5000, compress_level=9) + started = time.perf_counter() + assert status(big) == 201 + assert time.perf_counter() - started < 5 + + +def test_jpeg_header_is_parsed_incrementally(): + data = imagegen.jpeg(320, 200) + assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet + assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200) + + +@has_djpeg +@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"]) +def test_valid_jpeg(data): + assert status(data) == 201 + + +@has_djpeg +@pytest.mark.parametrize("name,data", [ + ("no scan data after SOS", imagegen.jpeg_without_scan_data()), + ("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]), +], ids=lambda v: v if isinstance(v, str) else "") +def test_broken_jpeg_is_415(name, data): + assert status(data) == 415, name + + +# ------------------------------------------------------------------ filename +@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨‍👩‍👧.png", " x ", "..."[:2] + "x"]) +def test_good_filenames(name): + assert validation.filename_param([("filename", name)]) == name + + +@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png", + "tab\t.png", "del\x7f.png", "x
y", "photo‮gnp.exe", "a" * 256]) +def test_bad_filenames(name): + with pytest.raises(InvalidRequest): + validation.filename_param([("filename", name)]) + + +def test_filename_must_be_single_and_present(): + with pytest.raises(InvalidRequest): + validation.filename_param([]) + with pytest.raises(InvalidRequest): + validation.filename_param([("filename", "a.png"), ("filename", "b.png")]) + + +def test_query_must_be_valid_utf8(): + assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")] + with pytest.raises(InvalidRequest): + validation.query_items(b"filename=%FF%FE.png") diff --git a/student-tests/unit/test_unit.py b/student-tests/unit/test_unit.py new file mode 100644 index 00000000..adf227ff --- /dev/null +++ b/student-tests/unit/test_unit.py @@ -0,0 +1,225 @@ +"""Pure functions, no services: validation, cursor, Authorization parsing, Argon2id, config.""" +from __future__ import annotations + +import asyncio +import base64 +import datetime as dt + +import pytest + +from messenger import authhdr, cursor, passwords, validation +from messenger.config import ConfigError, Settings +from messenger.errors import InvalidRequest, Unauthorized +from messenger.timefmt import UTC, from_us, rfc3339, to_us + + +def b64(raw: bytes) -> str: + return base64.b64encode(raw).decode() + + +# ------------------------------------------------------------------ password rules +@pytest.mark.parametrize("password", ["Twelve-chars", "я" * 128, "пароль:с:двоеточиями", "😀" * 12, "a b c d e f g"]) +def test_valid_passwords(password): + assert validation.password_field({"password": password}) == password + + +@pytest.mark.parametrize("password", ["short-11ch", "я" * 129, " " * 12, "\t\n " * 5, "\ud800" * 12, None, 123456789012]) +def test_invalid_passwords(password): + with pytest.raises(InvalidRequest): + validation.password_field({"password": password}) + + +def test_text_may_contain_nul_but_not_lone_surrogates(): + assert validation.text_field({"t": "a\u0000b"}, "t", 10) == "a\u0000b" + with pytest.raises(InvalidRequest): + validation.text_field({"t": "a\ud800"}, "t", 10) + + +# ------------------------------------------------------------------ cursor +CHAT = "3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11" + + +def test_cursor_roundtrip_and_binding(): + key = (to_us(dt.datetime(2026, 10, 12, 12, 0, 0, 123456, tzinfo=UTC)), "11111111-1111-4111-8111-111111111111") + token = cursor.encode(cursor.MESSAGES, CHAT, key) + assert cursor.decode(token, cursor.MESSAGES, CHAT) == key + with pytest.raises(InvalidRequest): + cursor.decode(token, cursor.MESSAGES, "00000000-0000-4000-8000-000000000000") # other chat + with pytest.raises(InvalidRequest): + cursor.decode(token, cursor.CHATS, CHAT) # other resource + + +@pytest.mark.parametrize("payload", [ + b"v1|m|" + CHAT.encode() + b"|253402300800000000|11111111-1111-4111-8111-111111111111", # 1 us after 9999-12-31 + b"v1|m|" + CHAT.encode() + b"|9999999999999999999|11111111-1111-4111-8111-111111111111", # int64 edge + b"v1|m|" + CHAT.encode() + b"|-1|11111111-1111-4111-8111-111111111111", + b"v1|m|" + CHAT.encode() + b"|1|not-a-uuid", + b"v1|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-11111111111G", + b"v2|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-111111111111", +]) +def test_broken_cursors_are_400_not_500(payload): + token = base64.urlsafe_b64encode(payload).rstrip(b"=").decode() + with pytest.raises(InvalidRequest): + cursor.decode(token, cursor.MESSAGES, CHAT) + + +def test_time_helpers_are_exact(): + moment = dt.datetime(2026, 2, 28, 23, 59, 59, 999999, tzinfo=UTC) + assert from_us(to_us(moment)) == moment + assert rfc3339(moment) == "2026-02-28T23:59:59.999999Z" + assert from_us(cursor.MAX_TS_US).year == 9999 + + +# ------------------------------------------------------------------ Authorization header +def test_basic_is_parsed_and_password_may_contain_colons(): + assert authhdr.parse_basic("Basic " + b64("alice:pa:ss:wörd".encode())) == ("alice", "pa:ss:wörd") + assert authhdr.parse_basic("basic " + b64(b"bob_1:x" * 1)) == ("bob_1", "x") + + +@pytest.mark.parametrize("header", [ + None, "", "Basic", "Basic é", "Basic !!!!", "Basic " + b64(b"no-colon"), "Basic " + b64(b"\xff\xfe:pw"), + "Basic " + b64(b":password"), "Basic " + b64(b"alice:"), "Basic " + b64("Алиса:pw".encode()), + "Bearer " + "a" * 43, "Basic " + "A" * 2000, +]) +def test_bad_basic_is_401_with_basic_challenge(header): + with pytest.raises(Unauthorized) as info: + authhdr.parse_basic(header) + assert info.value.headers["WWW-Authenticate"].startswith("Basic realm=\"messenger\"") + + +@pytest.mark.parametrize("header", [None, "", "Bearer", "Bearer short", "Bearer " + "a" * 43 + "!", + "Basic " + b64(b"alice:password-123"), "Token " + "a" * 43, "Bearer " + "é" * 43]) +def test_bad_bearer_is_401_with_bearer_challenge(header): + with pytest.raises(Unauthorized) as info: + authhdr.parse_bearer(header) + assert info.value.headers["WWW-Authenticate"] == 'Bearer realm="messenger"' + + +def test_bearer_ok(): + token = "A_b-" + "x" * 39 + assert authhdr.parse_bearer("Bearer " + token) == token + + +# ------------------------------------------------------------------ Argon2id +def test_argon2id_hash_and_verify(): + async def scenario(): + stored = await passwords.hash_password("я" * 128) + assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") + assert await passwords.verify_password(stored, "я" * 128) + assert not await passwords.verify_password(stored, "я" * 127) + assert not await passwords.verify_password(None, "anything") # unknown user, dummy hash + assert not await passwords.verify_password("not-a-hash", "anything") + assert await passwords.hash_password("same password!") != await passwords.hash_password("same password!") # salt + asyncio.run(scenario()) + + +# ------------------------------------------------------------------ config +@pytest.fixture(autouse=True) +def s3_credentials(monkeypatch): + """Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests.""" + monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST") + monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests") + + +def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + settings = Settings.from_env() + assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000) + assert settings.fault_points == frozenset() and settings.upload_concurrency == 4 + for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"), + ("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")): + monkeypatch.setenv(name, value) + with pytest.raises(ConfigError): + Settings.from_env() + monkeypatch.delenv(name) + monkeypatch.delenv("S3_SECRET_ACCESS_KEY") + with pytest.raises(ConfigError): + Settings.from_env() + + +def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch): + url_file = tmp_path / "database_url" + url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8") + monkeypatch.setenv("DATABASE_URL_FILE", str(url_file)) + monkeypatch.setenv("SESSION_STORE", "redis") + monkeypatch.setenv("CACHE_URL", "redis://:p@valkey:6379/0") + monkeypatch.setenv("SESSION_TTL_SECONDS", "2") + settings = Settings.from_env() + assert settings.database_url == "postgresql://u:p@db/x" + assert settings.session_store == "valkey" and settings.session_ttl_seconds == 2 + monkeypatch.setenv("SESSION_TTL_SECONDS", "0") + with pytest.raises(ConfigError): + Settings.from_env() + monkeypatch.setenv("SESSION_TTL_SECONDS", "2") + monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "missing")) + with pytest.raises(ConfigError) as info: + Settings.from_env() + assert "u:p" not in str(info.value) + + +# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id +@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"]) +def test_valid_idempotency_keys(key): + assert validation.idempotency_key(key) == key + + +@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"]) +def test_invalid_idempotency_keys(key): + with pytest.raises(InvalidRequest): + validation.idempotency_key(key) + + +@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}']) +def test_duplicate_keys_and_garbage_json_are_rejected(raw): + with pytest.raises(InvalidRequest): + validation.parse_json_object(raw) + + +def test_attachment_ids_absent_equals_empty_and_is_validated(): + assert validation.attachment_ids_field({"text": "x"}) == [] + assert validation.attachment_ids_field({"attachment_ids": []}) == [] + one = "11111111-1111-4111-8111-111111111111" + assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one] + for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11): + with pytest.raises(InvalidRequest): + validation.attachment_ids_field({"attachment_ids": bad}) + + +def test_request_fingerprint_ignores_key_order_and_default_attachments(): + from messenger.app import request_fingerprint + assert request_fingerprint("привет", []) == request_fingerprint("привет", []) + assert request_fingerprint("привет", []) != request_fingerprint("привет ", []) + assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", []) + one = "11111111-1111-4111-8111-111111111111" + assert request_fingerprint("x", [one]) != request_fingerprint("x", []) + + +def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch): + import socket + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.delenv("INSTANCE_ID", raising=False) + assert Settings.from_env().instance_id == socket.gethostname() + monkeypatch.setenv("INSTANCE_ID", "replica-a") + assert Settings.from_env().instance_id == "replica-a" + for bad in ("with space", "x" * 65, "ip/1"): + monkeypatch.setenv("INSTANCE_ID", bad) + with pytest.raises(ConfigError): + Settings.from_env() + + +def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600") + with pytest.raises(ConfigError): + Settings.from_env() + + +def test_message_write_path_setting(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.delenv("MESSAGE_WRITE_PATH", raising=False) + assert Settings.from_env().message_write_path == "function" + monkeypatch.setenv("MESSAGE_WRITE_PATH", "multi") + assert Settings.from_env().message_write_path == "multi" + monkeypatch.setenv("MESSAGE_WRITE_PATH", "fast") + with pytest.raises(ConfigError): + Settings.from_env()