ЛР 4 — БВТ2302 — Боданюк — на 5 #17

Closed
wfa wants to merge 11 commits from wfa/mtusi2026lab4:solution/bvt2302-bodanyuk into main
96 changed files with 25878 additions and 19 deletions
+48 -2
View File
@@ -5,16 +5,62 @@ APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
# The files are created by `python scripts/init_secrets.py` and mounted by compose as secrets.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`.
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas).
API_REPLICAS=2
# Idempotency-Key results are kept at least this long (contract: >= 24 h).
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
# Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path.
MESSAGE_WRITE_PATH=function
# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on
# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler.
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once.
UPLOAD_CONCURRENCY=4
UPLOAD_QUEUE_TIMEOUT_SECONDS=5
UPLOAD_READ_TIMEOUT_SECONDS=30
IMAGE_VERIFY_TIMEOUT_SECONDS=10
IMAGE_VERIFY_CONCURRENCY=1
JPEG_MAX_MEMORY_MB=256
S3_TIMEOUT_SECONDS=3
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
PENDING_GRACE_SECONDS=900
JANITOR_INTERVAL_SECONDS=60
# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty.
FAULT_POINTS=
# Additional settings of this implementation (defaults shown; all optional):
LOG_LEVEL=info
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_TIMEOUT_SECONDS=3
CACHE_TIMEOUT_SECONDS=0.5
PASSWORD_HASH_CONCURRENCY=2
DOCS_ENABLED=true
# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests.
DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=8
# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica.
API_CPUS=1.0
API_MEMORY=512m
S3_CPUS=1.0
S3_MEMORY=1g
PG_CPUS=2.0
PG_MEMORY=1g
VALKEY_CPUS=0.5
LB_CPUS=1.0
PG_MAX_CONNECTIONS=100
# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing)
APP_PUBLISH_PORT=8080
LB_STATS_PORT=8404
+5
View File
@@ -0,0 +1,5 @@
# LF everywhere: identical files on Windows/Linux and inside `git archive | docker build -`.
* text=auto eol=lf
*.png binary
# cmd.exe expects CRLF in batch files.
*.cmd text eol=crlf
+38
View File
@@ -0,0 +1,38 @@
name: Application (lab4)
on: [push, pull_request]
permissions:
contents: read
jobs:
app:
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Synthetic secrets and env (CI only, never committed)
run: python scripts/init_secrets.py && cp .env.example .env
- name: Build, migrate once, bootstrap the S3 bucket, start 2 API replicas behind HAProxy
run: docker compose --env-file .env up -d --build --wait
- name: Contract and public smoke
run: make check test PYTHON=python
- name: Persistence across restart of LB, API replicas, PostgreSQL, Valkey and S3
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python
- name: Failover of the replica that served the test, then restore it
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python
sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2
- name: Reconciler (track 2) dry-run on the stand - read-only report
run: docker compose run --rm reconciler
- name: Own tests (unit + stand; every failure is restored in finally)
run: |
python -m pip install --require-hashes --no-deps -r requirements.txt
python -m pip install -r requirements-test.txt
make student-test PYTHON=python
- name: Logs
if: always()
run: docker compose logs --tail 100 api lb s3 s3-init
- name: Remove CI stand and its synthetic data
if: always()
run: docker compose down -v
+66
View File
@@ -0,0 +1,66 @@
# syntax=docker/dockerfile:1
# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml
# (services api, migrate, s3-init, reconciler: one image, different commands).
#
# Python is interpreted, so there is no separate compile stage; the image
# contains only the interpreter, locked dependencies and the source code.
# Base image is pinned by version AND digest (python 3.13.16, Debian 12 bookworm).
FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
# 1. Dependencies first (layer cache survives source edits). Every package,
# including transitive ones, is pinned with sha256 hashes in requirements.txt;
# --no-deps guarantees nothing outside the lock file is installed.
COPY requirements.txt ./
RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt
# 2. JPEG integrity check: libjpeg-turbo's djpeg (strict mode, see src/messenger/images.py).
# Version pinned to the Debian 12 package; apt lists are removed in the same layer.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
&& rm -rf /var/lib/apt/lists/*
# 3. Unprivileged runtime user. Code stays owned by root, so the process
# cannot modify its own code.
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
COPY src/ ./src/
# Versioned SQL migrations, applied by `python -m messenger.migrate` (compose service migrate).
COPY migrations/ ./migrations/
# Course contract, served read-only at /openapi.json for Swagger UI (/docs).
COPY contracts/openapi.json ./contracts/openapi.json
# Non-secret defaults; compose.yaml passes the rest. Passwords come only as files
# (DATABASE_URL_FILE / CACHE_URL_FILE -> Docker secrets), never as ENV or COPY.
ENV PYTHONPATH=/app/src \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_TIMEOUT_SECONDS=8 \
DOCS_ENABLED=true \
SESSION_TTL_SECONDS=3600 \
DATABASE_URL_FILE=/run/secrets/database_url
# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out
# image distinguishable in `docker inspect` and in the "starting" log line of each replica.
ARG BUILD_REVISION=dev
LABEL org.opencontainers.image.revision=$BUILD_REVISION
ENV APP_REVISION=$BUILD_REVISION
USER app:app
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-m", "messenger.healthcheck"]
# Exec form: the Python server is PID 1 and receives SIGTERM directly (no shell in between).
CMD ["python", "-m", "messenger"]
+67
View File
@@ -28,3 +28,70 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---------------------------------------------------------------------------
# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile
secrets:
$(PYTHON) scripts/init_secrets.py
# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy.
up: secrets
docker compose up -d --build --wait
# Safe stop: containers stop, named volumes (data) stay.
down:
docker compose down
logs:
docker compose logs --tail 50 api lb
ps:
docker compose ps -a
migrate:
docker compose run --rm migrate
# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id.
lb-status:
$(PYTHON) scripts/lbctl.py status
$(PYTHON) scripts/lbctl.py replicas
# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP.
restore:
sh scripts/restore-api.sh
$(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2}
# Replace every API replica one by one with a freshly built image (no downtime).
rolling-update:
sh scripts/rolling-update.sh
# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
failover-series:
$(PYTHON) bench/failover_series.py --output evidence/failover
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
student-test:
$(PYTHON) -m pytest student-tests -v
unit-test:
$(PYTHON) -m pytest student-tests -m "not stack" -q
explain:
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only).
reconcile:
docker compose run --rm reconciler
reconcile-apply:
docker compose run --rm reconciler --apply
# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
bench-memory:
$(PYTHON) bench/upload_memory.py --output evidence/memory
# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/).
bench-reconcile:
$(PYTHON) bench/reconcile_cost.py --output evidence/reconcile
+236 -17
View File
@@ -1,37 +1,256 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Боданюк Алексей, БВТ2302
- Целевая оценка: **5**, трек 2 «Согласованность двух хранилищ»
- Ветка: `solution/bvt2302-bodanyuk`. Форк: `https://gitea.privcloud.tech/wfa/mtusi2026lab4`. PR: «ЛР 4 — БВТ2302 — Боданюк — на 5» в mtusi2026/mtusi2026lab4
- Стек: Python 3.13.16, Starlette 1.7.0, uvicorn 0.54.0, psycopg 3.3.6 + psycopg-pool 3.3.3, redis-py 8.1.0, argon2-cffi 25.1.0, **aiobotocore 3.9.2** (S3), **libjpeg-turbo 2.1.5 `djpeg`** (проверка JPEG); PostgreSQL 18.6, Valkey 8.1.3, HAProxy 3.2.26, **SeaweedFS 4.46**. Все образы закреплены по тегу **и** digest, все Python-пакеты — по sha256 ([requirements.txt](requirements.txt)), Debian-пакет — по версии ([Dockerfile](Dockerfile))
- Источники: README, COURSE, контракт ([openapi.json](contracts/openapi.json), [семантика](contracts/README.md)), документация S3 (PutObject/Content-MD5, ListObjectsV2), SeaweedFS (`weed server`, s3.json identities), PNG (RFC 2083 / W3C PNG 3rd ed.: chunks, CRC, zlib, фильтры, Adam7), libjpeg-turbo (`djpeg -strict`, `-scale`, `-maxmemory`), HAProxy 3.2 (`nbsrv`, `use_backend`, `http-request set-timeout`), cgroup v2 (`memory.peak`, `memory.events`). Учтены все замечания преподавателя к PR lab4 других студентов ([раздел ниже](#проверки-на-которых-ловили-другие-работы))
- Помощь ИИ: код, тесты, скрипты экспериментов и черновик отчёта написаны с помощью ИИ-ассистента Claude Code (Anthropic). Все команды и замеры реально выполнены на стенде ниже; вывод — в [evidence/](evidence/). Ответы на вопросы защиты — [docs/DEFENSE.md](docs/DEFENSE.md), согласованность хранилищ — [docs/CONSISTENCY.md](docs/CONSISTENCY.md), запуск на Windows и сценарий защиты — [docs/WINDOWS.md](docs/WINDOWS.md)
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Git, Docker с Compose v2 (проверено: Docker 29.8.0, Compose 5.5.1), Python 3.9+ и GNU make (на Windows — см. [WINDOWS.md](docs/WINDOWS.md)). Учётные данные в Git не хранятся, их создаёт скрипт.
```sh
git clone https://gitea.privcloud.tech/wfa/mtusi2026lab4 mtusi2026lab4 && cd mtusi2026lab4 && git checkout solution/bvt2302-bodanyuk
python3 scripts/init_secrets.py # пароли PostgreSQL/Valkey и ТРИ пары ключей S3 -> ./secrets (git-ignored), идемпотентно
cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
# postgres, valkey, s3 healthy -> migrate (0001..0004) и s3-init (bucket + проверка прав) exit 0 -> api x2 healthy -> lb
docker compose logs s3-init # anonymous GET/LIST -> 403, права ключей api/reconciler -> OK
make check && make test
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh && make restore
make reconcile # трек 2: dry-run, read-only отчёт JSON
```
- **Windows:** публичным тестам нужен `PYTHONUTF8=1`; хуки — через `scripts/restart.cmd` и `scripts/stop-one.cmd` (тот же `.sh` в Git Bash).
- **Свои тесты:** `python3 -m venv .venv`, `pip install --require-hashes --no-deps -r requirements.txt`, `pip install -r requirements-test.txt`, затем `make student-test` (~25 мин, стенд должен быть поднят). Без стенда — `make unit-test`.
- **Все доказательства одной командой** (на чистом clone, ~70 мин): `python scripts/collect_evidence.py`.
- **Bootstrap S3** — одноразовый сервис `s3-init` (`python -m messenger.s3_bootstrap`), идемпотентный: существующий bucket не трогает, sentinel-объект перезаписывает, затем проверяет модель доступа и **валит стенд** (exit 1), если bucket не приватный или у ключей лишние права. Повтор: `docker compose run --rm s3-init`.
- **Остановка:** безопасно — `docker compose down` (volumes `pgdata`, `valkeydata`, `s3data` сохраняются). Намеренное удаление учебных данных — `docker compose down -v` и `rm -rf secrets`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
клиент ──127.0.0.1:8080──> lb: HAProxy 3.2 ──edge──> api × 2 (Starlette, без состояния и без volume)
│ backend api: /health/ready (PG + Valkey + S3) │
│ backend api_core: /health/ready/core (PG + Valkey) — только если ├─data (internal)──> postgres 18.6: users, chats, messages, attachments (метаданные)
│ в api нет ни одного UP (S3 упал у всех) │ valkey 8.1.3: сессии
└ upload: timeout server 60 s └─storage (internal)─> s3: SeaweedFS 4.46, bucket messenger-attachments (байты)
migrate ──data──> 0001…0004 s3-init ──storage──> bucket, sentinel, проверка прав reconciler (по требованию) ──data+storage──> сверка
```
| Роль | Service | Образ | Health / readiness | Volume | Сеть / порт | Credentials | CPU / RAM |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Точка входа | `lb` | `haproxy:3.2.26-alpine@sha256:254e6370…` | `haproxy -c` + stats; бэкенды — активные проверки | — | edge; `127.0.0.1:8080`, `[::1]:8080`, `127.0.0.1:8404` | нет | 1.0 / 128m |
| API × 2 | `api` | `mtusi-messenger:lab4` из исходников | `/health/ready` (PG, Valkey, S3) | — | edge + data + storage, только `expose 8080` | `database_url`, `cache_url`, **своя** пара S3 (Read+Write bucket) | 1.0 / 512m, без swap |
| Миграции | `migrate` | тот же образ | exit 0 | — | data | `database_url` | 0.5 / 256m |
| Bootstrap S3 | `s3-init` | тот же образ | exit 0 | — | storage | admin + api + reconciler (для проверки прав) | 0.5 / 256m |
| БД | `postgres` | `postgres:18.6-bookworm@sha256:afc7e2d4…` | `pg_isready` | `pgdata` | data | `postgres_password` | 2.0 / 1g |
| Сессии | `valkey` | `valkey/valkey:8.1.3-bookworm@sha256:fea8b3e6…` | `valkey-cli ping` | `valkeydata` | data | `valkey_password` | 0.5 / 256m |
| Объекты | `s3` | `chrislusf/seaweedfs:4.46@sha256:08d51613…` | `GET /healthz` | `s3data` | storage, `8333` (S3 API) — не опубликован | identities из secrets ([start.sh](infra/seaweedfs/start.sh)) | 1.0 / 1g |
| Сверка (трек 2) | `reconciler` (профиль `tools`) | тот же образ | — | — | data + storage | `database_url`, пара S3 reconciler (+List) | 1.0 / 256m |
**Что перенесено из lab3** (отдельный коммит `d916d09`, без изменений): API, миграции 0001–0003, HAProxy без sticky с re-resolve DNS, общая идемпотентность, graceful shutdown, сессии в Valkey, все тесты lab3. **Что сделано в lab4:**
- **Выбор S3: SeaweedFS 4.46.** Репозитория `minio/minio` в Docker Hub больше нет (проверено при выборе), SeaweedFS активно развивается и поддерживает identity с правами на bucket. Один контейнер (`weed server`: master + volume + filer + S3 gateway). Web-интерфейсы master/filer/volume и их gRPC слушают **только 127.0.0.1 внутри контейнера** (`-ip.bind`), в сеть `storage` выходит лишь S3 API (8333/18333). Сеть internal, на хосте ничего не опубликовано. Телеметрия выключена, встроенный IAM API выключен, автосоздание bucket выключено.
- **Права.** Три пары ключей (`init_secrets.py`): `admin` (создание bucket) — только у `s3` и `s3-init`; `messenger-api` — `Read`+`Write` **на один bucket**, без `List` и `Admin`; `messenger-reconciler` — плюс `List`. Анонимной identity нет, поэтому bucket приватный. `s3-init` проверяет всё это при каждом запуске, включая канареечный bucket: API не видит его в ListBuckets и не может читать из него (403).
- **Модель данных** ([0004_attachments.sql](migrations/0004_attachments.sql), [DATA_MODEL.md](docs/DATA_MODEL.md)): `attachments` (метаданные; `object_key = o/<128 случайных бит>` назначает сервер; `status pending|ready`; карантин `lost_at/lost_reason`), `message_attachments` (FK + обратный поиск), `messages.attachment_ids uuid[]` (порядок, чтение истории остаётся одним index range scan). Отправка сообщения — SQL-функция `post_message_v2`, тоже одна транзакция.
- **Загрузка** ([attachments.py](src/messenger/attachments.py), [images.py](src/messenger/images.py)): `auth → членство (404) → filename / Content-Type / Content-Length (400/415/413) → слот загрузки → тело кусками с дедлайном → формат и пиксели по первым байтам → полная проверка → pending → PUT → ready → 201`. Подробно о порядке и сбоях — [CONSISTENCY.md](docs/CONSISTENCY.md).
- **Проверка содержимого.** PNG — свой потоковый валидатор: сигнатура, CRC **каждого** chunk, порядок chunk'ов, zlib-поток IDAT распаковывается порциями по 1 MiB и сразу выбрасывается, его длина должна быть ровно `height × (1 + байт в строке)` (Adam7 — по проходам), байт фильтра каждой строки ≤ 4, IEND в конце и ничего после него. Растр в памяти не создаётся. JPEG — `djpeg -strict -scale 1/8 -maxmemory`: эталонный декодер в отдельном процессе декодирует весь файл, любое предупреждение libjpeg — ошибка. Pillow для этого не годится: JPEG без данных после SOS он «успешно» декодирует в серую картинку (проверено). MIME определяется по байтам; несовпадение с `Content-Type` — 415.
- **Лимиты до чтения/декодирования.** `Content-Length > 10 MiB` → 413 без чтения тела; без длины (chunked) — 413 при пересечении лимита. Пиксели — из IHDR/SOFn по первым байтам: 20000×10000 → 413 через 33 байта, ничего не декодируется. Тело — один буфер ровно `Content-Length` байт (без копий до S3 и djpeg). Одновременно на реплику: `UPLOAD_CONCURRENCY=4` тела (остальные ждут ≤ 5 с с байтами в TCP-буфере, затем 503 + `Retry-After`), `IMAGE_VERIFY_CONCURRENCY=1` полная проверка. Дедлайны: тело 30 с, проверка 10 с, S3 3 с (+ время на объём), в HAProxy для загрузки `timeout server 60s`.
- **Скачивание** — только через API, без redirect: членство → `GET` объекта → стриминг кусками по 64 KiB (память не зависит от размера). Заголовки: настоящий `Content-Type`, `Content-Length`, `Content-Disposition: attachment` (ASCII-fallback + RFC 5987), `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `ETag` = SHA-256.
- **Готовность и деградированный режим.** `/health/ready` включает S3 (загрузка и скачивание — обязательный синхронный путь): S3 недоступен → 503. S3 — общая зависимость, и если он упал у всех, HAProxy переключается на backend `api_core` (проверка `/health/ready/core`: PG + Valkey). Чаты и текст работают, запросы к вложениям API сам отвечает 503 за ≤ 3 с. Если S3 потеряла одна реплика, она DOWN в `api`, трафик идёт на другую.
- **Janitor** (в каждой реплике, раз в ~60 с, advisory-замок): `pending` старше `PENDING_GRACE_SECONDS=900` → под `FOR UPDATE SKIP LOCKED` удалить объект, затем строку.
- **Reconciler (трек 2)** — [reconcile.py](src/messenger/reconcile.py), раздел «Трек 2» ниже.
- **`FAULT_POINTS`** (`after_s3_put`, `after_ready_commit`) — внедрение отказа ровно между шагами записи, только для экспериментов. По умолчанию пусто; неизвестное значение — ошибка конфигурации.
Readiness-критичные зависимости: PostgreSQL, Valkey и S3. LB в деградированном режиме обходится без S3.
Соответствие `.env.example` реальной конфигурации ([config.py](src/messenger/config.py), [compose.yaml](compose.yaml)):
| Переменная | Значение | Где используется |
| --- | --- | --- |
| `APP_HOST`, `APP_PORT`, `APP_ENV`, `SESSION_TTL_SECONDS`, `DATABASE_URL_FILE`, `SESSION_STORE`, `CACHE_URL_FILE`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS` | как в lab3 | без изменений |
| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | `http://s3:8333`, `us-east-1`, `messenger-attachments` | клиент S3 API, s3-init, reconciler; bucket создаёт s3-init |
| `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` | `/run/secrets/s3_access_key_id`, `…/s3_secret_access_key` | у API это **его** пара (secrets `s3_api_*` смонтированы под этими именами), у reconciler — его пара |
| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | `10485760`, `40000000` | лимиты контракта; можно уменьшить, увеличить нельзя (ConfigError) |
| `UPLOAD_CONCURRENCY`, `UPLOAD_QUEUE_TIMEOUT_SECONDS`, `UPLOAD_READ_TIMEOUT_SECONDS` | `4`, `5`, `30` | слоты загрузки на реплику, ожидание слота, дедлайн тела |
| `IMAGE_VERIFY_CONCURRENCY`, `IMAGE_VERIFY_TIMEOUT_SECONDS`, `JPEG_MAX_MEMORY_MB` | `1`, `10`, `256` | одновременные полные проверки, дедлайн проверки, `djpeg -maxmemory` |
| `S3_TIMEOUT_SECONDS` | `3` | дедлайн каждого вызова S3 (PUT — плюс время на объём) |
| `PENDING_GRACE_SECONDS`, `JANITOR_INTERVAL_SECONDS` | `900`, `60` | janitor незавершённых загрузок |
| `FAULT_POINTS` | пусто | внедрение отказов в экспериментах |
| `API_CPUS/MEMORY`, `S3_CPUS/MEMORY`, `PG_*`, `VALKEY_CPUS`, `LB_CPUS` | 1.0/512m, 1.0/1g, 2.0/1g, 0.5, 1.0 | лимиты контейнеров |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Стенд: Windows 10 22H2, AMD Ryzen 5 3400G (4 ядра / 8 потоков), 30 ГБ RAM, Docker Desktop 29.8.0 (WSL2), Python 3.14 на хосте, GNU make 4.4.1. Проверки выполнены 2026-10-25 **на стенде из чистого clone** (новая папка, `git clone`, пустые volumes, secrets созданы заново) одной командой `python scripts/collect_evidence.py`, коммит `f9f5ab9`. Этот прогон нашёл два дефекта (см. ниже). После исправления (коммиты `046e257`, `fc50362`, `6fb70de`) на том же стенде целиком повторены свои тесты, публичные `make test` / `test-persistence` / `test-failover` ×2 и замер стоимости обхода — [final-commit-public-checks.txt](evidence/final-commit-public-checks.txt).
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда / сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Чистый clone, secrets, миграции, bucket с нуля | `init_secrets.py`, `compose config --quiet`, `compose up -d --build --wait`, логи `migrate` и `s3-init` | 7 сервисов healthy / exit 0; `0001…0004: applied`; `bucket messenger-attachments: created`; образы по digest | [clean-clone.txt](evidence/clean-clone.txt) |
| Контракт | `make check`, `make validate` | PASS, PASS | [make-check-validate-test.txt](evidence/make-check-validate-test.txt) |
| **3:** публичный smoke (checksum, посторонний) | `make test` | `PASS: lab4 public API scenario` (и на финальном коммите) | там же, [final-commit-public-checks.txt](evidence/final-commit-public-checks.txt) |
| **3:** S3 в стенде; метаданные в PostgreSQL, байты в приватном bucket, не в БД и не на API | `test_metadata_reveals_no_storage_details`, `test_original_is_not_stored_on_api_containers`, политика compose | строка `ready` + объект `o/<hex>` того же размера в bucket; в ФС реплик нет файла такого размера; у `api` нет volume | [student-tests.txt](evidence/student-tests.txt) |
| **3:** 201 ready, оригинал побайтно, сообщение с `attachment_ids` читают участники | `test_png_and_jpeg_roundtrip_byte_for_byte`, `test_message_with_attachments_and_history`, `scripts/demo_attachments.py` | PNG и JPEG: SHA-256 и размер совпадают; `text=""` + файл → 201; история возвращает `attachment_ids` по порядку | там же |
| **3:** реальные PNG/JPEG ≤ 10 MiB и ≤ 40 Мпикс, MIME по содержимому, опасные filename, лимиты до чтения и декодирования | `test_size_limit_through_the_load_balancer`, `test_exactly_40_megapixels_is_accepted`, `test_pixel_limit_before_decoding`, `test_bad_content_is_415` (16 случаев), `test_bad_filename_is_400` (14) | ровно 10 485 760 байт через LB → 201, +1 байт → 413 за < 5 с, chunked сверх лимита → 413; 8000×5000 → 201; 20000×10000 / 8001×5000 / JPEG 400 Мпикс → 413 за < 2 с | там же |
| **3:** upload → остановка обслужившей реплики → download через другую | `test_upload_survives_failover_of_the_serving_replica`, `make test-failover` ×2 | после SIGKILL обслужившей 5 скачиваний с другой реплики — те же байты; публичный тест для `api-1` и `api-2`: 131 / 132 записи, **0 неудачных**, оригинал на месте | [test-failover.txt](evidence/test-failover.txt) |
| **3:** restart с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; новые PID и `StartedAt` у `lb`, `api-*`, `postgres`, `valkey`, **`s3`**; прежний токен, файл скачивается | [test-persistence.txt](evidence/test-persistence.txt) |
| **4:** тесты: большой файл, повреждённое изображение, подмена MIME, чужое `attachment_id`, файл другого чата; ошибки пользователя не дают 500 | `test_attachments.py` (56 тестов) | 413 / 415 / 400 / 404 по контракту; `Api.call` проверяет у **каждого** ответа формат ошибки, `X-Request-Id`, `X-Instance-Id`, отсутствие stack trace | [student-tests.txt](evidence/student-tests.txt) |
| **4:** прерывание загрузки и отказ между S3 и БД не оставляют вечных объектов; grace period | `test_consistency.py` (4), [CONSISTENCY.md](docs/CONSISTENCY.md) | обрыв и зависание тела — ничего не записано; `after_s3_put` → `pending` + объект, janitor убирает оба (grace 3 с в тесте, 900 с в стенде); `after_ready_commit` → ничего не удалено, файл скачивается; свежий `pending` не тронут | там же |
| **4:** RAM одной загрузки ограничена; ≥ 10 одновременных загрузок с пиком памяти | `bench/upload_memory.py` (16 и 32 клиента) | см. «Эксперименты»: пик +81…113 МБ и при 16, и при 32 клиентах (4 слота), лимит 512 МБ, 0 OOM процесса API | [memory/summary.json](evidence/memory/summary.json) |
| **4:** bootstrap идемпотентен, отдельные ограниченные ключи S3, консоль закрыта, проверка приватности вне API | логи `s3-init` (при каждом `up`), `test_bucket_is_private_outside_the_api`, `test_compose_policy_…` | anonymous GET/LIST → 403; ключ API: Read/Write да, List/CreateBucket/чужой bucket — нет; ключ reconciler: List да; повторный bootstrap — `exists, kept as is`; порты 9333/8888/8080 SeaweedFS недоступны из сети | [clean-clone.txt](evidence/clean-clone.txt), [student-tests.txt](evidence/student-tests.txt) |
| **4:** отказ S3 | `test_s3_outage_is_503_and_text_keeps_working[stop/pause]` | readiness 503 на всех репликах (core 200), upload/download 503 за < 8 с, метаданные 200, **текст работает** (деградированный режим LB); после восстановления файл цел | там же |
| **5:** трек 2 | `test_reconcile.py` (6), `bench/reconcile_faults.py`, `bench/reconcile_cost.py` | см. раздел «Трек 2» | [evidence/reconcile/](evidence/reconcile/) |
| Перенесённое из lab3 | все тесты lab3 (балансировка, идемпотентность, отказы БД/Valkey, explain) | проходят без ослабления; адаптированы только списки сервисов и миграций | [student-tests.txt](evidence/student-tests.txt) |
| Публичный генератор нагрузки | `tests/load.py --duration 30 --concurrency 4` | 5416 успешных, 0 ошибок, 180 req/s, p50/p95/p99 = 18 / 49 / 64 мс | [load.json](evidence/load.json) |
| Свои тесты | `python -m pytest student-tests -v` | **316 passed, 4 skipped** (113 unit + 203 против стенда, 21 мин); skipped — 4 unit-теста djpeg на хосте без libjpeg-turbo, те же случаи проходят на стенде | [student-tests.txt](evidence/student-tests.txt) |
**Что нашёл прогон из чистого clone (`f9f5ab9`).** 314 из 316 тестов прошли, два упали:
1. `test_s3_outage_is_503_and_text_keeps_working[stop]`. После stop/start S3 первое скачивание получало 503: в пуле aiobotocore оставались keep-alive соединения к старому процессу, а повторов не было. Исправлено одной повторной попыткой в том же дедлайне — все операции идемпотентны по ключу ([storage.py](src/messenger/storage.py), `046e257`).
2. `test_restart_does_not_extend_ttl`. Рестарт Valkey и обеих реплик (readiness теперь ждёт и S3) длился дольше 12-секундного TTL теста. TTL теста увеличен до 30 с.
Ещё дефекты нашли замеры памяти до этого прогона:
- семафор загрузок отпускался сразу после чтения тела, а буфер жил до конца S3 PUT. При 32 клиентах пик рос до +214 МБ при 4 слотах. Теперь слот держится до конца загрузки (`277d06f`);
- djpeg, убитый OOM-killer'ом, давал 415 вместо 503 — исправлено;
- PNG-валидатор копировал остаток IDAT на каждом шаге (`unconsumed_tail`) — исправлено.
И одну ошибку reconciler нашёл замер стоимости на том же стенде: `--apply` **без** `--verify-checksums` видел совпадающий размер и снимал карантин `checksum_mismatch` как `restored_object`, хотя сумму в этом режиме проверить не может. Теперь такой карантин снимает только проход с проверкой сумм (`6fb70de`). Тест дополнен, 6/6 тестов reconciler и smoke повторены на этом коммите — [reconcile-fix-rerun.txt](evidence/reconcile-fix-rerun.txt). Первый прогон своих тестов с двумя падениями сохранён как есть: [student-tests-first-run-f9f5ab9.txt](evidence/student-tests-first-run-f9f5ab9.txt).
## Проверки, на которых ловили другие работы
Замечания преподавателя к PR lab4 (#1–#13) других студентов, проверенные на своём стенде:
| Замечание к чужим PR | У нас | Где проверено |
| --- | --- | --- |
| PNG с корректным IHDR и мусорным IDAT (86 байт), обрезанный PNG, первые 33 байта fixture, повреждение после первых 64 KiB → 201 | 415: CRC каждого chunk, zlib до точного размера, IEND обязателен | `test_bad_content_is_415[png:*]`, unit `test_broken_png_is_415` |
| JPEG без данных после SOS → 201 | 415: `djpeg -strict` (Pillow такой файл принимает — проверено) | `test_bad_content_is_415[jpeg: no scan data after SOS]` |
| PNG 6–10 MiB → 413 HTML от nginx | HAProxy не ограничивает тело; ровно 10 485 760 байт → 201 через LB | `test_size_limit_through_the_load_balancer` |
| 20000×10000 → 500 (`DecompressionBombError`) | 413 по IHDR, ничего не декодируется; 40 000 000 px принимается, 40 005 000 → 413 | `test_pixel_limit_before_decoding`, `test_exactly_40_megapixels_is_accepted` |
| `img.load()` до проверки размера (пик 252 МБ на файле 131 КБ) | размер берётся из заголовка до чтения остального тела; растр PNG не создаётся вовсе | unit `test_png_verification_is_streaming_and_fast` (40 Мпикс < 5 с) |
| API получает admin-ключи MinIO и создаёт bucket | у API своя пара: CreateBucket → AccessDenied (проверяет `s3-init`) | [clean-clone.txt](evidence/clean-clone.txt), `test_compose_policy_…` |
| S3 остановлен: readiness 200 / 500 за 9 с, upload 500 / 504 | readiness 503, upload 503 за < 8 с (S3 3 с), без 500 | `test_s3_outage_…[stop/pause]` |
| filename с C1 (U+0085) или из пробелов `%20%20%20` → 201 | 400; также `/`, `\`, NUL, TAB, U+2028, bidi U+202E, `.`, `..`, невалидный UTF-8, повтор параметра | `test_bad_filename_is_400` (14) |
| `{"attachment_ids":[A]}`, `{"text":null,…}`, `{"text":"x","attachment_ids":null}` → 201 | 400 | `test_message_body_validation` (12) |
| COMMIT прошёл, ответ потерян → API удаляет объект, остаётся `ready` без байтов | в обработчике компенсации нет; janitor трогает только `pending` | `test_lost_answer_after_commit_never_deletes_the_object` |
| dry-run продолжает частичное удаление | dry-run read-only на уровне БД (`default_transaction_read_only`) и S3-обёртки; промежуточного состояния «deleting» нет | `test_detect_and_safely_fix_faults_on_both_sides` (отпечатки до/после) |
| тело накапливается целиком, рост RAM «не доказывает потоковость» | буфер ровно `Content-Length`, число буферов ограничено слотами; замер 16 и 32 клиентов | [memory/summary.json](evidence/memory/summary.json) |
| тест шлёт POST в чужой чат без `Idempotency-Key`; гонка считает 5xx конфликтом | все POST сообщений в своих тестах — с ключом; `Api.call` требует точный ожидаемый код | [conftest.py](student-tests/conftest.py) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Уровень 4: память при ≥ 10 одновременных загрузках
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
**Гипотеза:** память реплики ограничена `UPLOAD_CONCURRENCY × MAX_UPLOAD_BYTES` плюс одна полная проверка (`IMAGE_VERIFY_CONCURRENCY`) и не зависит от числа клиентов.
**Методика** ([bench/upload_memory.py](bench/upload_memory.py)):
- перед каждым сценарием реплики api пересоздаются: cgroup `memory.peak` начинается с простоя;
- N клиентов стартуют одновременно (barrier) через LB, у каждого свой файл;
- `memory.current` каждой реплики снимается каждые 50 мс внутри контейнера; затем читаются `memory.peak` (все процессы контейнера, включая djpeg), `VmHWM` процесса Python и `memory.events` (OOM-kill) всех контейнеров.
Лимит api: 1 CPU, 512 МБ, swap выключен (`memswap_limit`). Файлы: PNG 9,42 МБ из шума (не сжимается), JPEG 8000×5000 progressive 4:4:4 (0,6 МБ, самый дорогой для декодера: ~240 МБ буферов коэффициентов). Один прогон на сценарий, коммит `f9f5ab9`, сырые данные — [summary.json](evidence/memory/summary.json).
| Сценарий | Ответы | Пик реплик − простой (≈ 75 МБ) | Python VmHWM | OOM-kill |
| --- | --- | --- | --- | --- |
| PNG ×16, 4 слота (по умолчанию) | 16 × 201, p50 3,6 с | +81 / +89 МБ | 153 МБ | 0 |
| PNG ×**32**, 4 слота | 25 × 201, 7 × 503 «слишком много загрузок» (+Retry-After) | +80 / +113 МБ — **как при 16** | 144 / 181 МБ | 0 |
| PNG ×32, **16 слотов** | 24 × 201, 8 × 503 «хранилище недоступно» | +209 / +193 МБ | 286 / 267 МБ | 0 |
| JPEG 40 Мпикс ×16, verify=1 (по умолчанию) | 16 × 201 | +242 / +242 МБ (djpeg) | 108 МБ | 0 |
| JPEG 40 Мпикс ×16, **verify=4** | 2 × 201, 14 × 503 «не хватило ресурсов» | 512 / 512 МБ (лимит) | 111 / 92 МБ | 6 / 7 (djpeg) |
**Выводы:**
- Удвоение числа клиентов при тех же 4 слотах пик не меняет; лишние клиенты быстро получают 503 с `Retry-After`, а их байты ждут в TCP-буферах, а не в процессе.
- Повышение границы до 16 слотов поднимает пик в 2–2,5 раза и упирается уже в S3: SeaweedFS ограничивает одновременные загрузки, PUT не укладывается в дедлайн, клиент получает 503. Значит, 4 слота — разумная граница при 1 CPU.
- Полная проверка JPEG — самая дорогая часть. При одной проверке за раз пик +242 МБ укладывается в 512 МБ. При четырёх cgroup убивает djpeg; API это переживает, отвечает 503 (а не 415 — исправлено после первого замера) и не теряет ни процесс, ни другие запросы.
- S3 (SeaweedFS, лимит 1 ГБ, `GOMEMLIMIT=600MiB`): anon-память 386–412 МБ; `memory.current` до 1 ГБ — это page cache записанных томов.
**Ограничения.** Один хост и один прогон на сценарий (разброс между прогонами в моих повторах — до ±25 МБ). Клиенты на том же хосте. Python не возвращает освобождённую память ОС мгновенно, поэтому VmHWM — верхняя оценка.
### Отказы
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `docker compose stop s3` | readiness 503, вложения 503, текст работает | так и есть, загрузка 503 за < 8 с; HAProxy на `api_core` | сразу после старта S3 (с повтором соединения); потерь нет |
| `docker compose pause s3` | то же, ответ по дедлайну, а не зависание | 503 за ≤ 3 с + запас; readiness 503 | после unpause; потерь нет |
| сбой между PUT и COMMIT (`FAULT_POINTS=after_s3_put`) | 503, `pending` + объект, уборка после grace | так и есть, janitor убрал за ≤ 30 с при grace 3 с | мусор убран, `ready`-данные не затронуты |
| потерян ответ после COMMIT (`after_ready_commit`) | 503, но файл цел | `ready`, объект на месте, скачивается | ничего не удалено |
| клиент оборвал тело / застрял на середине | ничего не записано / 400 по дедлайну | 0 строк, 0 объектов; 400 с `X-Instance-Id` | — |
| SIGKILL реплики, обслужившей загрузку | другая реплика отдаёт файл | 5/5 скачиваний, те же байты | — |
| stop одной реплики (публичный тест) ×2 | ≤ 10 с, без потерь и дублей | 0 неудачных записей, ~6 с до первой записи после хука (время самого `docker stop` с drain) | — |
| restart всего стенда с volumes | данные, сессия и оригинал на месте | PASS, все 6 долгоживущих контейнеров с новым PID | — |
## Трек 2: согласованность двух хранилищ
**Гипотеза.** Один повторяемый проход, который merge-join'ит упорядоченный листинг S3 и keyset-страницы `attachments` по `object_key`, находит все расхождения обеих сторон. Он безопасно исправляет их (dry-run ничего не меняет; свежие и используемые вложения не удаляются; повторный запуск идемпотентен), а стоит O(N) по запросам и O(1) по памяти. Устройство и правила — [CONSISTENCY.md](docs/CONSISTENCY.md), код — [reconcile.py](src/messenger/reconcile.py).
**Методика.** Ошибки внедряются напрямую в хранилища: в S3 — изнутри сети `storage` ключами API/reconciler, в PostgreSQL — через psql. Затем запускается reconciler, сохраняется каждый JSON-отчёт. Повторяется тестом ([test_reconcile.py](student-tests/test_reconcile.py), 6 тестов) и сценарием с сохранением отчётов ([bench/reconcile_faults.py](bench/reconcile_faults.py) → [evidence/reconcile/](evidence/reconcile/)). `--grace 30 --allow-small-grace`, чтобы «старое» и «свежее» различались за секунды.
**Результаты, сценарий A** (все виды ошибок; [faults-summary.json](evidence/reconcile/faults-summary.json)):
| Внедрено | dry-run | `--apply` | повторный `--apply` | состояние после |
| --- | --- | --- | --- | --- |
| старый объект без строки | `orphan_object`, planned | удалён | — | объекта нет |
| `pending` старше grace с объектом / без | `stale_intent` ×2 | удалены (объект, затем строка) | — | нет ни строк, ни объекта |
| объект удалён под **используемым** `ready` (C) | `missing_object`, used | карантин `missing_object` | — | строка и ссылка из сообщения целы; content 404; прикрепить — 409 |
| объект обрезан под неиспользуемым `ready` (D) | `size_mismatch` | карантин | — | content 404 |
| тот же размер, другие байты (F, только `--verify-checksums`) | `checksum_mismatch` | карантин | — | content 404 |
| ложный карантин у здорового объекта (E) | `restored_object` | карантин снят | — | content 200 |
| свежий объект без строки, свежий `pending` (загрузка в процессе) | `fresh_*`, not_actionable | **не тронуты** | удалены, когда стали старше grace | — |
| здоровые A (используется), B | нет находок | не тронуты | — | content 200 |
| объект C возвращён «из бэкапа» | — | — | `restored_object` → карантин снят | content 200, те же байты |
- Dry-run: отпечаток всех строк `attachments` и список ключей bucket до и после совпали (`dry_run_changed_nothing: true`).
- Повторный `--apply`: 0 действий по исправленным ключам.
**Сценарий B — прерванный `--apply`.** 2000 старых сирот; S3 ставится на паузу во время удаления:
- прерванный проход: exit 1, `store_unavailable`, успел удалить 440;
- после unpause осталось 1560;
- повторный запуск: exit 0, удалено 1562 (1560 этих + 2 сироты, оставшиеся от тестов);
- третий запуск: 0 запланированных действий;
- [interrupted-*.json](evidence/reconcile/).
Ни одна `ready`-строка за весь эксперимент не удалена.
**Тесты безопасности:**
- `test_safety_valve_refuses_mass_deletion`: при `--max-orphan-fraction` ниже доли сирот выход 3, ничего не удалено;
- `test_store_outage_stops_the_run_without_damage[s3/postgres]`: выход 1 с JSON-отчётом;
- `test_live_uploads_are_never_hurt_by_concurrent_apply`: 40 загрузок + сообщений идут параллельно с повторяющимся `--apply --grace 8`. Все 40 вложений `ready`, без карантина, байты на месте.
**Стоимость обхода** ([cost.json](evidence/reconcile/cost.json)) — синтетические согласованные вложения по 1 КиБ, dry-run, медиана 3 запусков, reconciler 1 CPU / 256 МБ:
| Синтетических вложений (всего объектов в bucket) | Режим | Длительность, медиана из 3 | Запросы листинга S3 | Страницы БД | Прочитано байтов |
| --- | --- | --- | --- | --- | --- |
| 1 000 (1 285) | dry-run, страница 1000 | 0,30 с | 2 | 2 | 0 |
| 10 000 (10 285) | dry-run, страница 1000 | 1,67 с | 11 | 11 | 0 |
| 10 000 (10 285) | dry-run, страница 100 | 2,28 с (+37 %) | 103 | 103 | 0 |
| 10 000 (10 285) | dry-run, `--verify-checksums` | 48,4 с | 11 | 11 | 694 МБ (≈ 14 МБ/с) |
| 50 000 (50 285) | dry-run, страница 1000 | 8,4 с | 51 | 51 | 0 |
Время растёт линейно: ≈ 0,17 мс на объект, ≈ 6 000 объектов/с при 1 CPU. Число запросов ровно ⌈N/страница⌉ с каждой стороны. Страница 100 даёт в 10 раз больше запросов и +37 % времени, поэтому по умолчанию страница 1000 — максимум ListObjectsV2. Проверка сумм стоит O(объёма): 694 МБ — это в основном 10-мегабайтные файлы предыдущих тестов на стенде, а не 1-килобайтная синтетика. Поэтому `--verify-checksums` — режим для редкого (например, ночного) прогона, а сверка по листингу — для частого. Память reconciler от N не зависит (лимит 256 МБ, страница хранится одна).
**Ограничения трека:**
- Время берётся из часов БД (строки) и хоста (LastModified объектов). Поэтому grace должен быть много больше расхождения часов; при одном хосте это выполняется.
- Без `--verify-checksums` порча с тем же размером не видна; проверка сумм читает все байты и стоит O(общего объёма).
- Reconciler не знает, «правильный» ли объект вернули из бэкапа: он проверяет размер и SHA-256 по метаданным, а значит, восстановить можно только тот же оригинал.
- Карантин — решение для оператора, не автоматическое восстановление: байты потеряны.
- Листинг S3 и чтение БД — не единый снимок. Гонки закрывают grace и перепроверка под `FOR UPDATE`, а не изоляция.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Один Docker-хост, один экземпляр SeaweedFS** без репликации (`000`): отказ диска хоста теряет и БД, и bucket. Бэкап `s3data` и `pgdata` должен быть согласованным по времени; reconciler найдёт расхождение после восстановления, но байты не вернёт.
- **SeaweedFS 4.46**: S3 gRPC-порт 18333 слушает сеть `storage` вместе с 8333 (отдельного bind для него нет); сеть internal, в ней только api, s3-init и reconciler. KEK для SSE-S3 не задан: шифрования «на диске» нет (в lab4 не требуется, вопрос lab7).
- **HTTP внутри стенда**: Basic и ключи S3 идут без TLS — допустимо для локального стенда с синтетикой, TLS/mTLS — lab7.
- **Тело загрузки в памяти** (ограниченно): потоковой передачи в S3 через multipart нет. Причина — полная проверка изображения до записи требует весь файл. Граница — слоты; lab5 перенесёт обработку в staging + worker.
- **Повторная загрузка того же файла** создаёт новое вложение (контракт это разрешает); дедупликации по SHA-256 нет.
- **Проверка JPEG** принимает файлы, которые libjpeg декодирует без единого предупреждения. Редкие реальные файлы с «extraneous bytes» от кривых кодеков получат 415 — сознательная строгость.
- **Janitor** работает в репликах API (одна за раз, advisory-замок); при большой доле мусора его стоило бы вынести в отдельный сервис, как reconciler.
- **Замеры** — один хост и по одному прогону на сценарий памяти; выводы сделаны по крупным, повторяющимся различиям, а не по процентам.
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
for detection, only the active health check can find it).
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
Timeline of one round (all times from the moment the stop command is issued):
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
has every replica UP (rejoin time) -> 6 s steady state.
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
request (+s), the first moment after which the stopped replica never answered again
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
traffic again, plus the history check (no confirmed message lost, no duplicate).
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
<output>/series.md (table for REPORT.md). No tokens are written.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import json
import platform
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def docker(*args: str) -> str:
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
load = Load(workers).start()
time.sleep(6)
t_stop = load.now()
if mode == "pause":
docker("pause", replica["id"])
else:
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
t_stopped = load.now()
time.sleep(12)
t_start = load.now()
docker("unpause" if mode == "pause" else "start", replica["id"])
lbctl.wait_up(replicas, timeout=90)
t_up = load.now()
# The restarted container keeps its hostname: wait until it serves requests again.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
break
time.sleep(0.1)
t_serves = load.now()
time.sleep(6)
load.stop()
t_end = load.now()
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
rows = load.per_second()
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
for row in rows:
row = dict(row)
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
writer.writerow(row)
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
return {
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
"stop_command_seconds": round(t_stopped - t_stop, 2),
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
"failed_after_stop": len(failed_after_stop),
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
"target_last_served_after_stop_s": round(out_of_rotation, 3),
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
"client_retries_same_key": load.retries,
"history": load.verify_history(),
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/failover")
parser.add_argument("--modes", default="stop,kill,pause")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
assert len(replicas) >= 2, "need at least two running replicas"
lbctl.wait_up(len(replicas))
results = []
for mode in args.modes.split(","):
for replica in replicas: # each replica in turn; the previous one is already restored
print(f"== {mode} {replica['name']} ...", flush=True)
result = one_round(replica, mode, len(replicas), args.workers, out)
results.append(result)
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
"retry-on conn-failure + redispatch",
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8")
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for r in results:
h = r["history"]
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
*lines, ""]), encoding="utf-8")
print("\n".join(lines))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env python3
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
exactly as a correct client must do; the history is checked afterwards for losses and
duplicates. Every request is recorded as (t, kind, status, instance, latency).
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
"""
from __future__ import annotations
import argparse
import base64
import json
import math
import os
import threading
import time
import uuid
from collections import Counter, defaultdict
import urllib.error
import urllib.request
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
class Http:
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
def __init__(self, base_url: str, timeout: float):
import http.client
from urllib.parse import urlsplit
parts = urlsplit(base_url)
self.host, self.port = parts.hostname, parts.port or 80
self.timeout = timeout
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
self.conn = self.factory()
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
try:
self.conn.request(method, path, body=body, headers=headers or {})
response = self.conn.getresponse()
data = response.read()
if response.getheader("connection", "").lower() == "close":
self.conn.close()
self.conn = self.factory()
return response.status, dict(response.getheaders()), data
except Exception:
self.conn.close()
self.conn = self.factory()
raise
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
data = json.dumps(body).encode() if body is not None else None
h = {"Content-Type": "application/json", **(headers or {})}
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, json.loads(r.read() or b"null")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def make_user(label: str) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"
password = "Load-password-" + uuid.uuid4().hex
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, (status, user)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, (status, session)
return {"id": user["id"], "token": session["token"]}
class Load:
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
self.workers = workers
self.timeout = timeout
self.pause = pause
self.alice, self.bob = make_user("load"), make_user("load")
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
headers={"Authorization": "Bearer " + self.alice["token"]})
assert status == 201, chat
self.chat = chat
self.stop_event = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str, float]] = []
self.confirmed: dict[str, dict] = {}
self.retries = 0
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
def now(self) -> float:
return time.monotonic() - self.t0
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
with self.lock:
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
def _worker(self, index: int) -> None:
http = Http(BASE_URL, self.timeout)
auth = {"Authorization": "Bearer " + self.alice["token"]}
path = f"/api/v1/chats/{self.chat['id']}/messages"
n, key = 0, uuid.uuid4().hex
text = f"w{index}-0-{key[:6]}"
while not self.stop_event.is_set():
started = time.monotonic()
try:
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance, data = 0, "-", b""
self._record("POST", status, instance, time.monotonic() - started)
if status == 201:
msg = json.loads(data)
with self.lock:
self.confirmed[msg["id"]] = msg
n += 1
key = uuid.uuid4().hex
text = f"w{index}-{n}-{key[:6]}"
elif status in (0, 502, 503, 504, 500):
with self.lock:
self.retries += 1 # same key + same text on the next iteration
started = time.monotonic()
try:
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance = 0, "-"
self._record("GET", status, instance, time.monotonic() - started)
if self.pause:
time.sleep(self.pause)
def start(self) -> "Load":
self.t0 = time.monotonic()
for t in self.threads:
t.start()
return self
def stop(self) -> None:
self.stop_event.set()
for t in self.threads:
t.join(timeout=15)
def verify_history(self) -> dict:
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
items, cursor = [], None
while True:
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
assert status == 200, page
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
ids = [m["id"] for m in items]
texts = [m["text"] for m in items]
lost = [i for i in self.confirmed if i not in set(ids)]
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
# ------------------------------------------------------------------ summaries
def window(self, start: float, end: float) -> dict:
ev = [e for e in self.events if start <= e[0] < end]
ok = [e for e in ev if e[2] in (200, 201)]
lat = sorted(e[4] for e in ok)
def pct(p: float):
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
span = max(end - start, 1e-9)
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
def per_second(self) -> list[dict]:
buckets: dict[int, list] = defaultdict(list)
for e in self.events:
buckets[int(e[0])].append(e)
rows = []
for second in sorted(buckets):
ev = buckets[second]
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
return rows
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--duration", type=float, default=20)
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
load = Load(args.workers).start()
time.sleep(args.duration)
load.stop()
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
Everything synthetic is deleted at the end (objects first, then rows).
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import statistics
import subprocess
import sys
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
SHA = hashlib.sha256(OBJECT).hexdigest()
PUT_SCRIPT = """
import asyncio, sys
from messenger.config import Settings, s3_settings
from messenger.storage import Storage
DATA = bytes(range(256)) * 4
async def main():
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
await s.open()
keys = sys.stdin.read().split()
sem = asyncio.Semaphore(32)
async def one(k):
async with sem:
await s.{op}(k{args})
await asyncio.gather(*(one(k) for k in keys))
await s.close()
print(len(keys))
asyncio.run(main())
"""
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
encoding="utf-8", timeout=timeout, check=check)
def psql(sql: str) -> str:
"""SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB)."""
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
"ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip()
def s3_bulk(op: str, keys: list[str]) -> None:
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
def owner_and_chat() -> tuple[str, str]:
import base64
c = httpx.Client(base_url=BASE, timeout=30)
name = "rc_" + uuid.uuid4().hex[:10]
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
token = c.post("/api/v1/auth/sessions", headers={
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "bench", "member_ids": [me["id"]]}).json()
return me["id"], chat["id"]
def add(n: int, owner: str, chat: str) -> None:
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
for i in range(0, n, 5000):
part = keys[i:i + 5000]
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) "
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
s3_bulk("put", keys)
def reconcile(*args: str) -> dict:
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
return json.loads(out)
def measure(label: str, args: list[str], repeats: int) -> dict:
runs = [reconcile(*args) for _ in range(repeats)]
durations = [r["duration_seconds"] for r in runs]
last = runs[-1]
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
p.add_argument("--repeats", type=int, default=3)
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
owner, chat = owner_and_chat()
present = 0
results = []
steps = sorted(args.steps)
variant_step = steps[min(1, len(steps) - 1)]
try:
for target in sorted(args.steps):
started = time.perf_counter()
add(target - present, owner, chat)
present = target
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
if present == variant_step: # page-size and checksum variants at one size
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
("dry-run, --verify-checksums", ["--verify-checksums"])):
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
finally:
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
f"'{chat}'").split()
if keys:
s3_bulk("delete", keys)
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
print(f"removed {len(keys)} synthetic attachments", flush=True)
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
unused ready row, same-size corruption (found only with --verify-checksums);
DB side: stale intents (with / without object), quarantine mark on a healthy row;
fresh: orphan + intent younger than --grace.
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
lost object from a "backup" -> apply (quarantine lifted).
Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them ->
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
third run finds nothing.
python bench/reconcile_faults.py --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
s3_put)
from test_reconcile import insert_intent, snapshot # noqa: E402
GRACE = ["--grace", "30", "--allow-small-grace"]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def reconcile(*args: str) -> tuple[int, dict]:
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
return r.returncode, json.loads(r.stdout)
def save(out: Path, name: str, data) -> None:
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
def ours(report: dict, keys: dict[str, str]) -> dict:
"""The findings of this scenario's keys only (the stand also holds other test data)."""
f = findings_by_key(report)
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
for label, k in keys.items()}
def scenario_a(out: Path) -> dict:
api = Api()
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")}
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for n, a in up.items()}
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
"same-size corruption F": key["F"]}
orphan = "o/" + uuid.uuid4().hex
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(orphan, SAMPLE)
s3_put(stale_with, SAMPLE)
insert_intent(chat, alice, stale_with, SAMPLE, 600)
insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"])
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
time.sleep(32)
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
before = snapshot()
_, dry = reconcile("--verify-checksums", *GRACE)
after = snapshot()
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
s3_put(key["C"], data["C"]) # the lost object restored from a backup
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
save(out, name, report)
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
present = s3_keys()
summary = {
"dry_run_changed_nothing": before == after,
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
"after restore of C": ours(apply3, keys)},
"objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()},
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
"content_status_after": content,
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
apply3["duration_seconds"]],
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def scenario_b(out: Path, count: int = 2000) -> dict:
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
s3_exec("ks = sys.stdin.read().split()\n"
"sem = asyncio.Semaphore(32)\n"
"async def one(k):\n"
" async with sem:\n"
" await s.put(k, b'orphan', 'image/png', {})\n"
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
time.sleep(32)
result: dict = {}
def run_apply() -> None:
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
t = threading.Thread(target=run_apply)
t.start()
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
left = len(s3_keys() & set(keys))
if left < count:
break
time.sleep(0.2)
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
t.join()
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
time.sleep(5) # let S3 settle after unpause before counting
left_after_interrupt = len(s3_keys() & set(keys))
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
code3, third = reconcile("--apply", *GRACE)
save(out, "interrupted-1-apply.json", result["report"])
save(out, "interrupted-2-rerun.json", second)
save(out, "interrupted-3-third.json", third)
summary = {
"orphans_injected": count,
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
"applied": result["report"]["actions"]["applied"]},
"orphans_left_after_interrupt": left_after_interrupt,
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
"orphans_left_at_end": len(s3_keys() & set(keys)),
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
a = scenario_a(out)
b = scenario_b(out)
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
return 0
if __name__ == "__main__":
sys.exit(main())
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB.
python bench/rolling_under_load.py --output evidence/rolling-update
Writes <output>/summary.json (errors, RPS, history check, revisions before/after),
<output>/timeline.csv (per second ok/failed/instances) and <output>/script-output.txt.
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def revisions() -> dict[str, str]:
result = {}
for r in lbctl.replicas():
if r["state"] == "running":
label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}',
r["id"]], capture_output=True, text=True).stdout.strip()
result[r["name"]] = f"{r['hostname']} revision={label}"
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/rolling-update")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
before = revisions()
load = Load(args.workers).start()
time.sleep(5)
t_begin = load.now()
bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe")
if os.name == "nt" else "sh")
env = {**os.environ, "PYTHON": sys.executable}
script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
capture_output=True, text=True, encoding="utf-8", timeout=900)
t_end = load.now()
time.sleep(5)
load.stop()
(out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n"
+ script.stdout + script.stderr, encoding="utf-8")
with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
writer.writerows(load.per_second())
summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1),
"before": before, "after": revisions(),
"during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()),
"client_retries_same_key": load.retries, "history": load.verify_history(),
"workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"}
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")},
ensure_ascii=False, indent=2))
print(json.dumps(summary["during_update"], ensure_ascii=False))
return 0 if script.returncode == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
Method (repeatable, the stand must be up):
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
counter memory.peak starts from an idle process (it never decreases during a container's life);
* idle memory.current of each replica is read; then N clients upload at the same moment through
the LB (a barrier releases them together), each a different file;
* during the run memory.current of every replica is sampled every 50 ms inside the container;
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
PID 1 (the Python server alone) are read;
* the normal configuration is restored at the end.
python bench/upload_memory.py --output evidence/memory
"""
from __future__ import annotations
import argparse
import io
import json
import os
import platform
import statistics
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
MiB = 1024 * 1024
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
env={**os.environ, **(env or {})}).stdout
def replicas() -> list[str]:
return sh("docker", "compose", "ps", "-q", "api").split()
def cg(cid: str, name: str) -> int:
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
def memory_stat(cid: str, key: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines():
name, value = line.split()
if name == key:
return int(value)
return 0
def vmhwm(cid: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
if line.startswith("VmHWM:"):
return int(line.split()[1]) * 1024
return 0
def container_events() -> dict[str, str]:
"""restart count + OOM-kill events per container (cgroup memory.events oom_kill counter)."""
out = {}
for cid in sh("docker", "compose", "ps", "-q").split():
name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split()
events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines())
out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}"
return out
def recreate(env: dict) -> None:
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
time.sleep(3)
class Sampler(threading.Thread):
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
def __init__(self, cid: str):
super().__init__(daemon=True)
self.cid = cid
self.values: list[int] = []
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
stdout=subprocess.PIPE, text=True)
def run(self) -> None:
for line in self.proc.stdout:
if line.strip().isdigit():
self.values.append(int(line))
def stop(self) -> None:
self.proc.kill()
def actor(client: httpx.Client) -> tuple[str, str]:
name = "mem_" + uuid.uuid4().hex[:10]
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
import base64
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "mem", "member_ids": [me["id"]]}).json()
return token, chat["id"]
def big_jpeg(width: int, height: int, seed: int) -> bytes:
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
out = io.BytesIO()
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
return out.getvalue()
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
recreate(env)
ids = replicas()
idle = {cid: cg(cid, "memory.current") for cid in ids}
client = httpx.Client(base_url=BASE, timeout=120)
token, chat_id = actor(client)
barrier = threading.Barrier(len(files))
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
def upload(i: int, data: bytes, ctype: str) -> None:
c = httpx.Client(base_url=BASE, timeout=120)
barrier.wait()
started = time.perf_counter()
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
reason = r.json()["error"]["message"] if r.status_code != 201 else ""
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"), reason)
c.close()
s3_id = sh("docker", "compose", "ps", "-q", "s3").strip()
restarts_before = container_events()
samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)]
for s in samplers:
s.start()
time.sleep(0.5)
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
wall = time.perf_counter()
for t in threads:
t.start()
for t in threads:
t.join()
wall = time.perf_counter() - wall
time.sleep(0.5)
for s in samplers:
s.stop()
per_replica = {}
for cid, s in zip(ids, samplers[:-1]):
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
per_replica[hostname] = {
"idle_mib": round(idle[cid] / MiB, 1),
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
"uploads_served": sum(1 for r in results if r[2] == hostname),
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
}
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
- per_replica[hostname]["idle_mib"], 1)
lat = sorted(r[1] for r in results if r[0] == 201)
codes: dict[str, int] = {}
reasons: dict[str, int] = {}
for r in results:
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
if r[3]:
reasons[r[3]] = reasons.get(r[3], 0) + 1
summary = {
"scenario": name, "settings": env, "clients": len(files),
"s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1),
"s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1),
"s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1),
# OOM kills / restarts DURING the scenario (any container of the stand): must be empty
"incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)},
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
"types": sorted({t for _, t in files}), "status_codes": codes, "error_reasons": reasons, "wall_seconds": round(wall, 2),
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
"replicas": per_replica,
}
print(json.dumps(summary, ensure_ascii=False), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--clients", type=int, default=16)
p.add_argument("--output", default="evidence/memory")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
print("generating files ...", flush=True)
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)]
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
n, n2 = args.clients, 2 * args.clients
scenarios = [
(f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]),
(f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs),
(f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs),
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
]
results = []
sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process
sh("docker", "compose", "up", "-d", "--wait", "s3")
try:
for name, env, files in scenarios:
results.append(run_scenario(name, env, files))
finally:
recreate({}) # back to the compose defaults
meta = {
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
replicas()[0]).strip(),
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
}
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
print(json.dumps(meta, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())
+338
View File
@@ -0,0 +1,338 @@
# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`)
# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes).
# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md):
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...)
# docker compose --env-file .env up -d --build --wait
# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb.
# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and
# every service gets only the secrets it needs. Only the LB publishes ports (loopback only).
# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and
# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside).
name: mtusi-lab4
x-api-env: &api-env
APP_ENV: ${APP_ENV:-lab}
LOG_LEVEL: ${LOG_LEVEL:-info}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-valkey}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
DB_POOL_MIN: ${DB_POOL_MIN:-1}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5}
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
# Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests.
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8}
DOCS_ENABLED: ${DOCS_ENABLED:-true}
MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function}
# Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits.
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5}
UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30}
IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10}
IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1}
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
# Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures.
FAULT_POINTS: ${FAULT_POINTS:-}
services:
postgres:
image: postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
# pg_stat_statements: per-statement counters for the experiments.
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -q"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
restart: unless-stopped
networks: [data]
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
valkey:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
# The password goes into a config include readable only by the valkey user,
# so it never appears in the process arguments. docker-entrypoint.sh drops root.
command:
- sh
- -c
- >-
umask 077 &&
printf 'requirepass %s\n' "$$(cat /run/secrets/valkey_password)" > /tmp/requirepass.conf &&
chown valkey /tmp/requirepass.conf &&
exec docker-entrypoint.sh valkey-server /etc/valkey/valkey.conf
secrets: [valkey_password]
volumes:
- ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro
- valkeydata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"]
interval: 2s
timeout: 3s
retries: 30
start_period: 5s
restart: unless-stopped
networks: [data]
deploy:
resources:
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
# Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one
# process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities
# and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on
# 127.0.0.1 inside the container only.
s3:
image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62
entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
# Go heap target below the container limit: the GC returns memory before the cgroup OOM.
GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB}
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_api_access_key_id
- s3_api_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
volumes:
- ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro
- s3data:/data
healthcheck:
# Unauthenticated health endpoint of the S3 gateway (200 once it serves requests).
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 30s
restart: unless-stopped
networks: [storage]
deploy:
resources:
limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"}
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
migrate:
build:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: ${API_IMAGE:-mtusi-messenger:lab4}
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
restart: "no"
networks: [data]
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
# One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model
# (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3.
s3-init:
image: ${API_IMAGE:-mtusi-messenger:lab4}
command: ["python", "-m", "messenger.s3_bootstrap"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_api_access_key_id
- s3_api_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
depends_on:
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully # same image: built once by `migrate`
healthcheck:
disable: true
restart: "no"
networks: [storage]
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
api:
build:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: ${API_IMAGE:-mtusi-messenger:lab4}
environment: *api-env
secrets:
- database_url
- cache_url
- source: s3_api_access_key_id
target: s3_access_key_id
- source: s3_api_secret_access_key
target: s3_secret_access_key
# No `ports` and no container_name: replicas are reachable only through `lb`,
# and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname.
expose: ["8080"]
healthcheck:
test: ["CMD", "python", "-m", "messenger.healthcheck"]
interval: 2s
timeout: 2s
retries: 3
start_period: 5s
depends_on:
postgres:
condition: service_healthy
valkey:
condition: service_healthy
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
restart: unless-stopped
networks: [edge, data, storage]
deploy:
replicas: ${API_REPLICAS:-2}
resources:
# 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime;
# measured in evidence/memory/ (bench/upload_memory.py).
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"}
# No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound
# of the experiments is a real bound, and running out of it is visible, not hidden in swap.
memswap_limit: ${API_MEMORY:-512m}
lb:
image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
# The ONLY public entry point, on the loopback interfaces only. Both IPv4 and IPv6:
# on Windows `localhost` resolves to ::1 first, and a refused ::1 connection costs ~2 s
# per request before the fallback to 127.0.0.1 (measured in lab 3: 2.0 s vs 0.007 s).
- "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080"
- "[::1]:${APP_PUBLISH_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas)
healthcheck:
# Config is valid and the process answers; whether any API is ready is the LB's own job.
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"]
interval: 5s
timeout: 3s
retries: 3
depends_on:
api:
condition: service_healthy
restart: unless-stopped
networks: [edge]
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
# Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on
# demand, dry-run by default (read-only on both stores):
# docker compose run --rm reconciler # report only
# docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine
reconciler:
profiles: ["tools"]
image: ${API_IMAGE:-mtusi-messenger:lab4}
entrypoint: ["python", "-m", "messenger.reconcile"]
command: []
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
secrets:
- database_url
- source: s3_reconciler_access_key_id
target: s3_access_key_id
- source: s3_reconciler_secret_access_key
target: s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
restart: "no"
networks: [data, storage]
deploy:
resources:
limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m}
networks:
edge:
data:
internal: true # no route outside: stores talk only to api/migrate
storage:
internal: true # S3 API reachable only by api, s3-init, reconciler
volumes:
pgdata:
valkeydata:
s3data:
secrets:
postgres_password:
file: ./secrets/postgres_password
valkey_password:
file: ./secrets/valkey_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
s3_api_access_key_id:
file: ./secrets/s3_api_access_key_id
s3_api_secret_access_key:
file: ./secrets/s3_api_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
+73
View File
@@ -0,0 +1,73 @@
# Согласованность PostgreSQL и S3 (lab4: уровень 4 и трек 2)
Байты вложения лежат в приватном bucket S3 (SeaweedFS), метаданные — в таблице `attachments` PostgreSQL. Общей транзакции у двух хранилищ нет, поэтому порядок записи и механизм уборки определяют, что остаётся после любого сбоя.
## Инвариант и порядок записи
**Объект никогда не существует без строки, которая называет его ключ.** Строка появляется раньше объекта, а при уборке исчезает позже.
```text
upload: [проверки, тело в памяти, полная проверка изображения]
1. INSERT attachments(status='pending', object_key=o/<128 бит>) COMMIT ← намерение
2. S3 PUT o/<key> (Content-MD5)
3. UPDATE status='ready' WHERE status='pending' COMMIT
4. 201 ready
janitor: SELECT … WHERE status='pending' AND старше PENDING_GRACE FOR UPDATE SKIP LOCKED
S3 DELETE (идемпотентно) → DELETE строки COMMIT
```
| Сбой | Что остаётся | Кто и когда убирает | Что видит клиент |
| --- | --- | --- | --- |
| До шага 1 (лимит, 415, обрыв тела, таймаут) | ничего: до полной проверки в хранилища ничего не пишется | — | 4xx/обрыв |
| Шаг 1 не прошёл (БД недоступна) | ничего | — | 503 |
| Между 1 и 2, S3 PUT упал или неизвестно, записан ли объект | строка `pending` (+ возможно объект) | janitor через `PENDING_GRACE_SECONDS` (900 с): удаляет объект, затем строку | 503, повтор создаёт новое вложение (контракт это разрешает) |
| Между 2 и 3 (процесс упал, БД пропала) — `FAULT_POINTS=after_s3_put` | строка `pending` + объект | janitor | 503 |
| Шаг 3 закоммичен, ответ потерян — `FAULT_POINTS=after_ready_commit` | корректное `ready`-вложение | **никто не трогает**: janitor работает только с `pending` | 503; файл скачивается по id ([test_consistency.py](../student-tests/test_consistency.py)) |
В обработчике запроса **компенсации нет**. Если ответ на COMMIT потерян, неизвестно, стала ли строка `ready`. Удалить объект «на всякий случай» значило бы получить `ready` без байтов — именно это преподаватель нашёл в чужом PR. Решение принимает только janitor, по зафиксированному состоянию и под блокировкой строки: «зависшая» загрузка, если бы она ещё шла, ждала бы на этой строке и затем получила бы 503.
**Почему grace 900 с безопасен.** Самая долгая загрузка: ожидание слота 5 с + тело 30 с + проверка 10 с + S3 PUT ≈ 4 с, итого < 60 с. Grace на порядок больше. Reconciler отказывается работать с `--grace` < 60 с без явного флага `--allow-small-grace` (флаг нужен только тестам).
**Когда честно вернуть 201.** Только после шага 3: объект записан (S3 подтвердил PUT и проверил Content-MD5), строка `ready` закоммичена. До этого клиент не получает id.
## Reconciler (трек 2)
`python -m messenger.reconcile` — отдельный CLI в том же образе, compose-сервис `reconciler` (профиль `tools`, запускается по требованию). У него свои ключи S3 (`Read+Write+List` на bucket), у API права `List` нет.
### Как он обходит хранилища
Оба хранилища читаются как упорядоченные по ключу потоки, и над ними выполняется merge-join:
- S3: `ListObjectsV2(prefix=o/, StartAfter, MaxKeys=1000)` — ключи в порядке байтов;
- PostgreSQL: keyset-страницы `SELECT … FROM attachments WHERE object_key > $1 ORDER BY object_key LIMIT 1000` — колонка `COLLATE "C"`, тот же порядок байтов, уникальный индекс.
Память не зависит от размера bucket. Стоимость прохода ≈ N/1000 запросов листинга + N/1000 страниц БД; с `--verify-checksums` добавляется чтение всех байтов.
### Что находит и что исправляет `--apply`
| Вид | Условие | `--apply` |
| --- | --- | --- |
| `orphan_object` | объект без строки, старше `--grace` | удалить объект (перед этим заново проверить, что строки нет и объект всё ещё старый) |
| `stale_intent` | строка `pending` старше `--grace` (с объектом или без) | под `FOR UPDATE`: удалить объект, затем строку |
| `missing_object` | строка `ready`, объекта нет, старше `--grace` | **карантин** строки: `lost_at`, `lost_reason` |
| `size_mismatch` | у `ready` размер объекта ≠ `size_bytes` | карантин |
| `checksum_mismatch` | SHA-256 объекта ≠ сохранённому (`--verify-checksums`) | карантин |
| `restored_object` | строка в карантине, объект снова на месте и верный | снять карантин |
| `fresh_*` | то же, но моложе `--grace` (загрузка в процессе) | ничего |
Карантин: `GET …/content` → 404 «оригинал утрачен», прикрепить к новому сообщению нельзя (409, как failed-вложение). Метаданные и ссылки из старых сообщений сохраняются. Карантин обратим: когда объект восстановят из резервной копии, следующий проход его снимет.
### Почему исправление безопасно
1. **Dry-run read-only конструктивно, а не «по if-у».** Сессия PostgreSQL открывается с `default_transaction_read_only=on`: любая запись упадёт в самой БД. Объект S3 в dry-run обёрнут в `ReadOnlyStorage` без метода `delete`. Тест сравнивает отпечаток всех строк `attachments` и список ключей bucket до и после dry-run. Это закрывает ошибку, которую нашли в чужом PR: «dry-run продолжает частично завершённое удаление».
2. **`ready`-строки никогда не удаляются**, ни используемые, ни неиспользуемые: только обратимый карантин. Удаляются лишь объекты без строк и строки-намерения `pending`. Ссылки `message_attachments → attachments` защищены FK, поэтому используемое вложение не удалить даже по ошибке.
3. **Всё, что моложе `--grace`, не трогается.**
4. **Перепроверка перед каждым действием**, под `FOR UPDATE`, где есть строка. `post_message_v2` берёт `FOR SHARE` на прикрепляемые вложения, поэтому карантин и отправка сообщения сериализуются.
5. **Объект первым, строка второй.** Прерванный проход оставляет состояние, которое следующий проход доводит до конца. Повторный запуск идемпотентен.
6. **Предохранитель.** Если «сиротами» выглядят больше `--max-orphan-fraction` (50 %) объектов — например, reconciler направили не в ту БД, — `--apply` завершается с кодом 3, ничего не удалив.
7. **Отказ хранилища** во время прохода: выход с кодом 1 и JSON-отчётом `store_unavailable`; уже сделанное зафиксировано, остальное сделает следующий запуск.
### Janitor и reconciler — зачем оба
- **Janitor** (в каждой реплике API, раз в ~60 с, advisory-замок — работает одна реплика за раз) дёшев: частичный индекс по `pending` и ничего больше. Он убирает типичный мусор — незавершённые загрузки.
- **Reconciler** проходит оба хранилища целиком и находит то, о чём БД не знает: объекты без строк (ручные действия, баги прошлых версий, восстановление bucket из другой копии), пропавшие и испорченные объекты. Его запускают по расписанию или вручную, всегда сначала в dry-run.
+47
View File
@@ -0,0 +1,47 @@
# Модель данных lab4
Схема: [0001_init.sql](../migrations/0001_init.sql) (lab2), [0002_idempotency_keys.sql](../migrations/0002_idempotency_keys.sql) и [0003_post_message_function.sql](../migrations/0003_post_message_function.sql) (lab3), [0004_attachments.sql](../migrations/0004_attachments.sql) (lab4). Миграции применяет один раз отдельный сервис `migrate` до старта реплик API; миграции lab3 и lab4 только добавляют объекты (0004 ещё ослабляет `CHECK` длины текста), поэтому совместимы с работающей предыдущей версией. Байты вложений живут **только** в приватном bucket S3 (SeaweedFS), в PostgreSQL — их метаданные; согласованность двух хранилищ — [CONSISTENCY.md](CONSISTENCY.md). Источник истины для пользователей, чатов, участников и сообщений — PostgreSQL 18. Для сессий источник истины — хранилище, выбранное через `SESSION_STORE`. По умолчанию это Valkey с AOF, его можно переключить на таблицу `sessions` в PostgreSQL.
## Таблицы и ограничения
| Таблица | Ключ | Ограничения | Зачем |
| --- | --- | --- | --- |
| `users` | `id uuid` | `UNIQUE (username)`, `CHECK` формата username, длина `display_name` | уникальность имени держит БД, а не код: гонку регистрации разрешает уникальный индекс |
| `chats` | `id uuid` | FK `created_by → users`, длина `title` | `last_message_at` — «часы» чата для упорядочивания сообщений |
| `chat_members` | `(chat_id, user_id)` | FK на `chats` и `users`, `UNIQUE (chat_id, position)` | `position` хранит порядок `member_ids`; `chat_created_at` — копия времени чата для keyset-списка чатов |
| `messages` | `id uuid` | FK `(chat_id, sender_id) → chat_members`, длина `text` | отправитель обязан быть участником — это гарантирует сама БД |
| `sessions` | `token_hash bytea` | FK `user_id → users`, длина хеша 32 байта | хранится SHA-256 токена; сам токен не хранится нигде |
| `idempotency_keys` | `(user_id, chat_id, key)` | FK на `users`, `chats`; `message_id → messages` (DEFERRABLE, `UNIQUE`); `CHECK` формата ключа и длины SHA-256 | общая для всех реплик идемпотентность отправки: ключ → отпечаток тела → сообщение. Хранится ≥ `IDEMPOTENCY_TTL_SECONDS` (24 ч), очистка — пачками раз в 10 мин под advisory-замком |
| `attachments` (lab4) | `id uuid` | FK `(chat_id, owner_id) → chat_members`; `UNIQUE object_key` (`COLLATE "C"`, формат `o/<32 hex>`); `CHECK` статуса `pending/ready`, типа `image/png|jpeg`, размера 1…10 MiB, пикселей ≤ 40 млн, длины SHA-256, карантина (`lost_at`/`lost_reason` только у `ready`) | метаданные оригинала; `object_key` генерирует сервер (128 случайных бит), с id и filename не связан. `pending` — намерение загрузки (строка раньше объекта), `ready` — объект записан. Частичный индекс `(created_at) WHERE status='pending'` — для janitor |
| `message_attachments` (lab4) | `(message_id, position)` | FK на `messages` и `attachments`, `UNIQUE (message_id, attachment_id)`, индекс по `attachment_id` | целостность ссылок (вложение, на которое ссылается сообщение, удалить нельзя) и обратный поиск «используется ли вложение» |
| `schema_migrations` | `version` | checksum файла | какие миграции применены; изменённую задним числом миграцию мигратор не применит |
**Почему `bytea` для текста.** Контракт разрешает символ U+0000, а тип `text` в PostgreSQL не может содержать байт NUL (`DataError`). Поэтому `display_name`, `title` и `text` хранятся как байты UTF-8. Длину в code points проверяет API, а `CHECK` в байтах — страховка (1 code point ≤ 4 байта).
## Индексы и запросы
- `messages_chat_keyset (chat_id, created_at, id)`. Первая страница — `WHERE chat_id=$1 ORDER BY created_at, id LIMIT n`, следующая — `WHERE chat_id=$1 AND (created_at, id) > ($2, $3) ORDER BY ... LIMIT n`. Сравнение строк `(created_at, id) > (...)` попадает в `Index Cond`: поиск начинается прямо с позиции cursor и читает только `n+1` строк. Запросов два, а не один с `($2 IS NULL OR ...)`: иначе при generic plan условие cursor уходит в `Filter`, и поздняя страница сканирует всю историю чата до неё.
- `chat_members_user_keyset (user_id, chat_created_at, chat_id)` — список чатов пользователя тем же keyset-способом.
- `users_username_key` — вход по username и уникальность.
- `sessions_expires_at` — очистка истёкших сессий.
## Транзакционные границы
| Операция | Транзакция | Что гарантирует |
| --- | --- | --- |
| Регистрация | один `INSERT` | `UniqueViolation` → 409; ровно одна строка при гонке |
| Создание чата | `INSERT chats` → `INSERT chat_members` (все участники одним запросом) → `COMMIT` | неизвестный участник вызывает FK-ошибку **после** вставки чата; откат удаляет и чат, и частичное членство |
| Сообщение (lab3, с ключом) | проверка членства → `INSERT idempotency_keys … ON CONFLICT DO UPDATE … WHERE ключ старше 24 ч` → если вставлен: `UPDATE chats … RETURNING` → `INSERT messages`; если нет: прочитать сохранённое сообщение и сравнить отпечаток → `COMMIT`. По умолчанию всё это — одна SQL-функция `post_message` в autocommit (одно обращение к БД) | ключ, сообщение и «результат» коммитятся вместе или не коммитятся вовсе: «висящего» ключа нет. Конкурентный повтор ждёт на уникальном индексе завершения первой транзакции и затем видит её результат (READ COMMITTED, новый снимок на каждый оператор). Другое тело → 409, тот же — тот же Message |
| Сообщение с `attachment_ids` (lab4) | в той же функции `post_message_v2`: проверка членства → проверка вложений (`ready`, этого чата, не в карантине; `FOR SHARE`) → только если всё хорошо — захват ключа → `UPDATE chats` → `INSERT messages (…, attachment_ids)` → `INSERT message_attachments` | чужое/неизвестное/незавершённое вложение — 404, в карантине — 409, ключ при этом не «сгорает». Порядок `attachment_ids` хранится массивом в строке сообщения: страница истории остаётся одним диапазонным сканом индекса |
| Загрузка вложения (lab4) | `INSERT attachments … 'pending'` (autocommit) → S3 PUT (Content-MD5) → `UPDATE … SET status='ready' WHERE status='pending'` (autocommit) → 201 | 201 только когда и объект, и строка `ready` зафиксированы. Сбой между шагами оставляет `pending`, его убирает janitor после `PENDING_GRACE_SECONDS`. Компенсации в обработчике запроса нет: после потерянного ответа COMMIT неизвестно, `ready` ли строка |
| Сообщение (lab2-часть) | проверка членства → `UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + 1 мкс) RETURNING` → `INSERT messages` → `COMMIT` | строка чата заблокирована до конца транзакции, поэтому писатели одного чата идут по очереди. Время сообщений строго растёт **в порядке коммитов**: страница по cursor не пропустит сообщение, закоммиченное «задним числом» |
| Вход | `SELECT` пользователя, проверка Argon2id, запись сессии | неверный пароль ничего не пишет |
| Logout | `DELETE` сессии (или `DEL` в Valkey) | повтор с тем же токеном — 401 |
Чтение истории не требует snapshot между HTTP-запросами (так в контракте): новое сообщение появится на следующей странице. Список чатов сортируется по `chats.created_at = clock_timestamp()` без блокировки. Два чата, создаваемые одновременно, могут закоммититься не в порядке времени. Тогда у обхода, который уже начался, такой чат может оказаться «до» cursor; новый обход его увидит. Чаты создаются редко, поэтому мы не сериализуем их глобальным замком; это ограничение описано в REPORT.md.
## Сессии
- Токен: `secrets.token_urlsafe(32)` — 256 бит случайности, в ответе 43 символа. В хранилище лежит только SHA-256 токена. Для случайного 256-битного значения быстрого хеша достаточно, медленный KDF нужен только для низкоэнтропийных паролей.
- `SESSION_STORE=valkey` (по умолчанию): ключ `session:<sha256-hex>`, значение `{user_id, expires_at}`. Срок жизни задаётся абсолютным `PXAT expires_at`, поэтому рестарт его не продлевает. Включён `appendonly yes` + `appendfsync always`: подтверждённые вход и logout сохраняются до ответа клиенту. Valkey — **единственное** хранилище сессий, это не кэш над PostgreSQL. Поэтому нет второй копии, которая могла бы «оживить» отозванный токен после отказа. Если Valkey недоступен, проверить сессию нельзя: защищённые запросы и logout получают 503, анонимный доступ не открывается.
- `SESSION_STORE=postgres`: таблица `sessions`, проверка `expires_at > now()` в запросе.
+39
View File
@@ -0,0 +1,39 @@
# Ответы на вопросы защиты ЛР 4
## Почему файл в Docker volume одной API-реплики не равен объектному хранилищу?
- **Volume принадлежит одному контейнеру.** Файл, загруженный через реплику api-1, лежит только у неё. Запрос, который LB отправил на api-2, его не найдёт. Если api-1 остановлена (failover, rolling update, scale down), файл недоступен вовсе. В нашем стенде у сервиса `api` нет ни одного volume: реплики взаимозаменяемы, и тест «загрузка → kill обслужившей реплики → скачивание через другую» проходит ([test_storage_failures.py](../student-tests/test_storage_failures.py)).
- **Общий bind-mount на одном хосте** тоже не выход. Он привязывает все реплики к одной машине, не даёт атомарной записи объекта, проверки целостности (Content-MD5), отдельных прав доступа и HTTP-API для других сервисов (worker в lab5 возьмёт тот же объект по ключу).
- **S3-сервер** — отдельный процесс со своим жизненным циклом и своим volume (`s3data`): он переживает перезапуск и замену API. Права в нём задаются по ключам: у API — только Read+Write своего bucket, без List и без создания bucket.
## Когда можно честно вернуть 201?
Когда оба хранилища подтвердили запись: S3 ответил на PUT, проверив Content-MD5, и после этого закоммичен `UPDATE attachments SET status='ready'`. До этого клиент не получает id вложения. Перед записью проходят все проверки: лимиты, полная проверка изображения, SHA-256. Ответ 201 на «принял в память» или «записал строку, а PUT ещё идёт» был бы ложью: после сбоя по такому id ничего не скачать. Подробно: [CONSISTENCY.md](CONSISTENCY.md).
## Что произойдёт, если S3 PUT прошёл, а COMMIT в PostgreSQL — нет?
Строка-намерение `pending` с ключом объекта закоммичена **до** PUT, поэтому объект не «потеряется»: БД знает его ключ.
- Клиент получает 503 и может повторить: получится новое вложение, контракт это разрешает.
- Строка остаётся `pending`, вложение невидимо (GET → 404, прикрепить → 404).
- Janitor через `PENDING_GRACE_SECONDS` (900 с) под `FOR UPDATE` удаляет объект, затем строку. Проверено внедрением отказа `FAULT_POINTS=after_s3_put`: строка и объект исчезают ([test_consistency.py](../student-tests/test_consistency.py)).
- Если упал не COMMIT, а **ответ** на него (COMMIT прошёл), в обработчике ничего не удаляется: неизвестно, стала ли строка `ready`. Проверено `FAULT_POINTS=after_ready_commit`: файл цел и скачивается.
- Объекты, о которых БД не знает вовсе (ручные действия, старые версии), находит reconciler (трек 2) и удаляет, если они старше grace.
## Почему случайный object key и приватный bucket не заменяют ACL приложения?
- **Приватный bucket** запрещает анонимный доступ к S3, но решение «может ли *этот пользователь* читать *этот файл*» принимает только приложение: оно знает участников чата. Ключи S3 у API одни на всех пользователей. Поэтому каждый `GET /attachments/{id}` и `/content` проверяет членство в чате, посторонний получает 404 (не 403 — существование чужого вложения не раскрывается).
- **Случайный ключ** — защита от перебора, а не авторизация. Ключ может утечь (логи, отладка, бывший участник чата). Удалённого из чата пользователя в нашем API нет, но в общем случае доступ должен отзываться — а ключ, если бы он был единственной защитой, остался бы действующим навсегда. Поэтому ключ не попадает ни в один ответ API, а байты отдаются только через авторизованный API, без redirect на S3.
- То же проверено снаружи: анонимный GET/LIST к S3 → 403 (bootstrap `s3-init` и тест), а запрос с ключом API работает — значит, права проверяет S3, а доступ пользователей — приложение.
## Частые уточняющие вопросы
**Как ограничена память?** Одна загрузка — один буфер ≤ 10 MiB (при известном Content-Length выделяется ровно этот объём, без копий). На реплику одновременно принимается не больше `UPLOAD_CONCURRENCY=4` тел. Остальные ждут ≤ 5 с, их байты остаются в TCP-буферах (backpressure), потом 503 + Retry-After. Полная проверка изображения выполняется по одной (`IMAGE_VERIFY_CONCURRENCY=1`). Пиксели проверяются по заголовку ещё до чтения остального тела. Замер ≥ 10 одновременных загрузок: [REPORT.md](../REPORT.md#эксперименты-и-отказы).
**Почему не Pillow для проверки?** Для PNG Pillow создаёт растр целиком (40 Мпикс RGBA = 160 МБ), а наш валидатор проверяет CRC и zlib-поток потоково и ничего не хранит. Для JPEG Pillow игнорирует предупреждения libjpeg и возвращает серую картинку для файла без данных после SOS. `djpeg -strict` считает любое предупреждение ошибкой, а `-scale 1/8` декодирует все блоки, но пишет растр в 64 раза меньше.
**Что если S3 недоступен?** `/health/ready` → 503 (S3 входит в обязательный синхронный путь). Загрузка и скачивание → 503 за ≤ `S3_TIMEOUT_SECONDS` (3 с). HAProxy переключается на backend `api_core` с проверкой `/health/ready/core` (БД + Valkey): чаты и текстовые сообщения продолжают работать. Если S3 потеряла одна реплика, она DOWN в основном backend, и трафик идёт на другую.
**Почему SeaweedFS, а не MinIO?** Образы MinIO в Docker Hub больше не публикуются (репозиторий `minio/minio` удалён, проверено при выборе). SeaweedFS развивается и умеет identity с правами на bucket (`Read:bucket`, `Write:bucket`, `List:bucket`). Закреплён по digest (4.46, сентябрь 2026). Его web-интерфейсы (master, filer, volume) слушают только 127.0.0.1 внутри контейнера, телеметрия выключена.
**Где dry-run гарантирует, что ничего не меняется?** Не в коде планировщика, а в окружении: сессия PostgreSQL в dry-run открыта с `default_transaction_read_only=on`, объект S3 обёрнут без метода `delete`. Тест сравнивает отпечаток всей таблицы `attachments` и список ключей bucket до и после.
+20
View File
@@ -0,0 +1,20 @@
## Лабораторная 4
Автор / группа: Боданюк Алексей, БВТ2302
Целевая оценка (3/4/5): 5
Трек 5, если заявлен: трек 2 «Согласованность двух хранилищ» (reconciler S3 ↔ PostgreSQL)
Что реализовано и что перенесено: реализация lab3 перенесена отдельным коммитом без изменений. Добавлено: S3-хранилище SeaweedFS 4.46 (приватный bucket, идемпотентный bootstrap с проверкой прав, отдельные ключи для API / reconciler / admin), вложения PNG/JPEG (метаданные в PostgreSQL, байты в S3; полная проверка целостности до записи; лимиты до чтения и декодирования), `attachment_ids` в сообщениях, скачивание через API без redirect, деградированный режим LB при отказе S3, janitor незавершённых загрузок, reconciler с dry-run, внедрение отказов, замеры памяти и стоимости обхода. Подробно — REPORT.md.
Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`python3 scripts/init_secrets.py`, `cp .env.example .env`, `docker compose --env-file .env up -d --build --wait`).
Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев», вывод в evidence/.
Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги».
Источники и использование ИИ: REPORT.md, шапка (код, тесты и отчёт написаны с помощью Claude Code; все замеры реально выполнены).
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
+122
View File
@@ -0,0 +1,122 @@
# Запуск на ноутбуке с Windows и сценарий защиты ЛР 4
Команды — в **PowerShell** из папки проекта (где лежит `compose.yaml`). Переменные `$env:...` действуют до закрытия окна.
## 1. Что установить один раз
| Программа | Зачем | Важно |
| --- | --- | --- |
| [Git for Windows](https://git-scm.com/download/win) | clone и Git Bash | путь по умолчанию `C:\Program Files\Git`: оттуда `scripts\*.cmd` запускают `.sh`-хуки |
| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | PostgreSQL, Valkey, SeaweedFS (S3), HAProxy, API | backend WSL 2, виртуализация в BIOS; в Settings → Resources ≥ 4 CPU и ≥ 6 ГБ RAM |
| [Python 3.12+](https://www.python.org/downloads/windows/) | secrets, публичные и свои тесты, скрипты замеров | галочка **Add python.exe to PATH** |
| GNU make (необязательно) | `make check/test/test-failover …` | `winget install ezwinports.make`, затем **новое** окно PowerShell. Рецептам Makefile нужен `sh`: в каждом окне один раз `$env:Path += ";C:\Program Files\Git\usr\bin"` |
## 2. Развернуть проект (дома, заранее — нужен интернет)
```powershell
git clone https://gitea.privcloud.tech/wfa/mtusi2026lab4 mtusi2026lab4
cd mtusi2026lab4
git checkout solution/bvt2302-bodanyuk
python scripts\init_secrets.py # случайные пароли и 3 пары ключей S3 в .\secrets (не в Git)
Copy-Item .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
```
Первый запуск качает образы (~1 ГБ) и собирает API — 3–10 минут. Команда завершается, когда postgres, valkey, s3, обе реплики api и lb в состоянии `healthy`, а `migrate` и `s3-init` завершились с кодом 0. Вердикты bootstrap S3 (приватность, права ключей): `docker compose logs s3-init`.
Окружение для своих тестов и скриптов (один раз):
```powershell
python -m venv .venv
.venv\Scripts\python -m pip install --require-hashes --no-deps -r requirements.txt
.venv\Scripts\python -m pip install -r requirements-test.txt -r requirements-dev.txt
$env:PYTHONUTF8 = "1"
```
Если запущен стенд lab3, он занимает порт 8080: `docker compose -p mtusi-lab3 stop` (данные lab3 сохранятся).
## 3. Перед защитой
```powershell
cd mtusi2026lab4
$env:PYTHONUTF8 = "1"; $env:Path += ";C:\Program Files\Git\usr\bin"
docker compose up -d --wait
make check test PYTHON=python # или: python tests\smoke.py
python scripts\lbctl.py status # обе реплики UP
```
## 4. Сценарий защиты из README — по шагам
**Шаг 1. Загрузить fixture, отправить сообщение, скачать вторым участником, сравнить SHA-256.**
```powershell
.venv\Scripts\python scripts\demo_attachments.py
```
Скрипт печатает каждый запрос и ответ: регистрирует alice, bob и eve, создаёт чат alice+bob, загружает `tests/fixtures/sample.png` (201 ready, sha256), отправляет сообщение с `attachment_ids` и `text=""`. Затем bob скачивает файл, SHA-256 совпадает. Для eve (посторонний) — 404 на метаданные и на байты. Отдельно показано, что ключ объекта в S3 случайный и в API не раскрывается.
**Шаг 2. Прямой неаутентифицированный GET к S3 не отдаёт объект.** Скрипт из шага 1 в конце делает изнутри сети `storage` (снаружи S3 не опубликован) анонимный GET того же объекта и анонимный листинг bucket — оба 403. Тот же GET, подписанный ключом API, — 200. Вручную:
```powershell
docker compose logs s3-init # anonymous GET/LIST -> 403, права ключей api/reconciler
.venv\Scripts\python -m pytest student-tests\test_storage_failures.py -k private -v
```
**Шаг 3. Persistence и failover: общий тест сохраняет и заново читает оригинал через API.**
```powershell
make test-persistence ACTION_SCRIPT=scripts/restart.cmd PYTHON=python # restart lb, api, postgres, valkey, s3
docker compose ps # у всех «Up N seconds»
make test-failover ACTION_SCRIPT=scripts/stop-one.cmd PYTHON=python
python scripts\lbctl.py replicas # остановленная — exited
make restore PYTHON=python
```
Для второй реплики: сдвинуть round-robin одним запросом (`Invoke-WebRequest http://127.0.0.1:8080/health/live -UseBasicParsing | Out-Null`) и повторить `make test-failover`. Загрузка → kill обслужившей реплики → скачивание через другую: `.venv\Scripts\python -m pytest student-tests\test_storage_failures.py -k failover -v`.
**Шаг 4. На API нет локального файла оригинала.**
```powershell
.venv\Scripts\python -m pytest student-tests\test_attachments.py -k not_stored -v # find по ФС реплик
docker compose config --format json | Select-String '"volumes"' # у api нет volume
```
**Уровень 4 вживую.**
```powershell
# Отказ S3 (stop и pause): readiness 503, загрузка 503 за < 8 с, текст работает (деградированный режим LB):
.venv\Scripts\python -m pytest student-tests\test_storage_failures.py -k outage -v
# Сбой между PUT и COMMIT убирает janitor; потерянный ответ после COMMIT ничего не удаляет:
.venv\Scripts\python -m pytest student-tests\test_consistency.py -v
# Все негативные кейсы файлов (413/415/400, чужой чат, MIME):
.venv\Scripts\python -m pytest student-tests\test_attachments.py -v
```
**Трек 2 вживую.**
```powershell
docker compose run --rm reconciler # dry-run: отчёт JSON, ничего не меняет
.venv\Scripts\python -m pytest student-tests\test_reconcile.py -k both_sides -v -s # ошибки на обеих сторонах, ~1,5 мин
docker compose run --rm reconciler --apply # исправить (на чистом стенде — 0 действий)
```
Готовые отчёты прогонов — [evidence/reconcile/](../evidence/reconcile/), стоимость обхода — [evidence/reconcile/cost.json](../evidence/reconcile/cost.json).
**Все свои тесты** (~25 мин): `.venv\Scripts\python -m pytest student-tests -v`.
## 5. Если что-то пошло не так
| Симптом | Что сделать |
| --- | --- |
| `error during connect … dockerDesktopLinuxEngine` | Docker Desktop не запущен — запустить и подождать минуту |
| `port is already allocated` на 8080/8404 | занято стендом lab3 (`docker compose -p mtusi-lab3 stop`) или в `.env` поставить `APP_PUBLISH_PORT=8081`, `LB_STATS_PORT=8405` |
| `secrets/... not found` при `up` | не выполнен `python scripts\init_secrets.py` |
| `s3-init` exit 1 | `docker compose logs s3-init`: какая проверка доступа не прошла |
| `UnicodeDecodeError` в `tests\*.py` | не задан `$env:PYTHONUTF8 = "1"` |
| `*.cmd`: «не удаётся найти путь» | Git установлен не в `C:\Program Files\Git` — поправить путь в `scripts\*.cmd` |
| после `stop-one` в LB одна реплика | так и задумано; вернуть: `make restore PYTHON=python` |
| api `unhealthy` после сна ноутбука | `docker compose restart` и `python scripts\lbctl.py wait-up 2` |
| нужно начать с чистого листа | `docker compose down -v` (удаляет учебные данные, включая bucket) → `docker compose up -d --build --wait` |
Остановить стенд после защиты: `docker compose down` (данные сохранятся) или `docker compose down -v` (данные удаляются).
+294
View File
@@ -0,0 +1,294 @@
# Clean clone: secrets, config, build, migrations from zero, start, versions
# 2026-10-25T11:39:46Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов
$ git status --short --branch
## solution/bvt2302-bodanyuk...origin/solution/bvt2302-bodanyuk
[exit 0, 0.1 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe --version
Python 3.14.6
[exit 0, 0.0 s]
$ docker version --format client {{.Client.Version}} / server {{.Server.Version}}
client 29.8.0 / server 29.8.0
[exit 0, 0.1 s]
$ docker compose version
Docker Compose version v5.5.1
[exit 0, 0.1 s]
$ make --version
GNU Make 4.4.1
Built for Windows32
Copyright (C) 1988-2023 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
[exit 0, 0.1 s]
$ python scripts/init_secrets.py
created secrets/postgres_password
created secrets/valkey_password
created secrets/database_url
created secrets/cache_url
created secrets/s3_admin_access_key_id
created secrets/s3_admin_secret_access_key
created secrets/s3_api_access_key_id
created secrets/s3_api_secret_access_key
created secrets/s3_reconciler_access_key_id
created secrets/s3_reconciler_secret_access_key
[exit 0, 0.1 s]
$ cp .env.example .env
$ docker compose --env-file .env config --quiet
[exit 0, 0.2 s]
$ docker compose --env-file .env up -d --build --wait
#1 [internal] load local bake definitions
#1 reading from stdin 1.32kB 0.0s done
#1 DONE 0.0s
#2 [migrate internal] load build definition from Dockerfile
#2 transferring dockerfile: 2.90kB done
#2 DONE 0.0s
#3 [api] resolve image config for docker-image://docker.io/docker/dockerfile:1
#3 DONE 1.1s
#4 [migrate] docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e
#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.0s done
#4 CACHED
#5 [api internal] load metadata for docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
#5 DONE 0.0s
#6 [api internal] load .dockerignore
#6 transferring context: 196B done
#6 DONE 0.0s
#7 [migrate 1/9] FROM docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
#7 resolve docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 0.0s done
#7 DONE 0.1s
#8 [migrate internal] load build context
#8 transferring context: 420.34kB 0.1s done
#8 transferring context: 420.34kB 0.1s done
#8 DONE 0.1s
#9 [api 2/9] WORKDIR /app
#9 CACHED
#10 [api 5/9] RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 && rm -rf /var/lib/apt/lists/*
#10 CACHED
#11 [api 3/9] COPY requirements.txt ./
#11 CACHED
#12 [api 4/9] RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt
#12 CACHED
#13 [api 7/9] COPY src/ ./src/
#13 CACHED
#14 [api 6/9] RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
#14 CACHED
#15 [api 8/9] COPY migrations/ ./migrations/
#15 CACHED
#16 [api 9/9] COPY contracts/openapi.json ./contracts/openapi.json
#16 DONE 0.0s
#17 [api] exporting to image
#17 exporting layers
#17 exporting layers 0.1s done
#17 exporting manifest sha256:61c866ef0176db37bf16836248289c639db9adf4af46bbd1bbd6b5bf36f37d64 0.0s done
#17 exporting config sha256:51199f0c44f5a7bb410bb7446b9c1a569b46165515f46d4065fb1fcdb4bbeb73 0.0s done
#17 exporting attestation manifest sha256:05366afe97838405c3e894464a12615f917115bec85c16befdc5503003c1f470
#17 ...
#18 [migrate] exporting to image
#18 exporting layers 0.1s done
#18 exporting manifest sha256:4110f2c418eaa8d2334845ca16721650556be9cf998b3763e558d1677128333e 0.0s done
#18 exporting config sha256:fdc58a5045cb098687769b14073cec7a7867599a2eb441679c0ac97df186c22b 0.0s done
#18 exporting attestation manifest sha256:d356c4ccd69e747feceab14426981048b02a2b0bcba8a29ad66d24ffe553177c 0.0s done
#18 exporting manifest list sha256:cf5183dbd465657100545822d97a395adee71d05ac7e2b9d5f5f4fa25de424dc 0.0s done
#18 naming to docker.io/library/mtusi-messenger:lab4 done
#18 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
#18 DONE 0.3s
#17 [api] exporting to image
#17 exporting attestation manifest sha256:05366afe97838405c3e894464a12615f917115bec85c16befdc5503003c1f470 0.0s done
#17 exporting manifest list sha256:c438a3f95cfa71ecc8ddb2d7bda0b2f7a4b549ba73a3a45c523539d162c8f2de 0.0s done
#17 naming to docker.io/library/mtusi-messenger:lab4 done
#17 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
#17 DONE 0.4s
#19 [migrate] resolving provenance for metadata file
#19 DONE 0.0s
#20 [api] resolving provenance for metadata file
#20 DONE 0.0s
Image mtusi-messenger:lab4 Building
Image mtusi-messenger:lab4 Building
Image mtusi-messenger:lab4 Built
Image mtusi-messenger:lab4 Built
Network mtusi-lab4_edge Creating
Network mtusi-lab4_data Creating
Volume mtusi-lab4_s3data Creating
Network mtusi-lab4_edge Creating
Volume mtusi-lab4_pgdata Creating
Volume mtusi-lab4_pgdata Creating
Network mtusi-lab4_storage Creating
Network mtusi-lab4_storage Creating
Volume mtusi-lab4_s3data Creating
Network mtusi-lab4_data Creating
Volume mtusi-lab4_valkeydata Creating
Volume mtusi-lab4_valkeydata Creating
Volume mtusi-lab4_s3data Created
Volume mtusi-lab4_s3data Created
Volume mtusi-lab4_pgdata Created
Volume mtusi-lab4_pgdata Created
Volume mtusi-lab4_valkeydata Created
Volume mtusi-lab4_valkeydata Created
Network mtusi-lab4_edge Created
Network mtusi-lab4_edge Created
Network mtusi-lab4_storage Created
Network mtusi-lab4_storage Created
Container mtusi-lab4-s3-1 Creating
Network mtusi-lab4_data Created
Network mtusi-lab4_data Created
Container mtusi-lab4-valkey-1 Creating
Container mtusi-lab4-postgres-1 Creating
Container mtusi-lab4-valkey-1 Created
Container mtusi-lab4-postgres-1 Created
Container mtusi-lab4-migrate-1 Creating
Container mtusi-lab4-s3-1 Created
Container mtusi-lab4-migrate-1 Created
Container mtusi-lab4-s3-init-1 Creating
Container mtusi-lab4-s3-init-1 Created
Container mtusi-lab4-api-2 Creating
Container mtusi-lab4-api-1 Creating
Container mtusi-lab4-api-2 Created
Container mtusi-lab4-lb-1 Creating
Container mtusi-lab4-lb-1 Created
Container mtusi-lab4-api-1 Created
Container mtusi-lab4-postgres-1 Starting
Container mtusi-lab4-s3-1 Starting
Container mtusi-lab4-valkey-1 Starting
Container mtusi-lab4-postgres-1 Started
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-s3-1 Started
Container mtusi-lab4-valkey-1 Started
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-migrate-1 Starting
Container mtusi-lab4-migrate-1 Started
Container mtusi-lab4-s3-1 Waiting
Container mtusi-lab4-migrate-1 Waiting
Container mtusi-lab4-s3-1 Healthy
Container mtusi-lab4-migrate-1 Exited
Container mtusi-lab4-s3-init-1 Starting
Container mtusi-lab4-s3-init-1 Started
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-valkey-1 Waiting
Container mtusi-lab4-s3-1 Waiting
Container mtusi-lab4-migrate-1 Waiting
Container mtusi-lab4-s3-init-1 Waiting
Container mtusi-lab4-migrate-1 Exited
Container mtusi-lab4-s3-1 Healthy
Container mtusi-lab4-valkey-1 Healthy
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-s3-init-1 Exited
Container mtusi-lab4-api-1 Starting
Container mtusi-lab4-api-1 Started
Container mtusi-lab4-api-2 Starting
Container mtusi-lab4-api-2 Started
Container mtusi-lab4-api-1 Waiting
Container mtusi-lab4-api-2 Waiting
Container mtusi-lab4-api-1 Healthy
Container mtusi-lab4-api-2 Healthy
Container mtusi-lab4-lb-1 Starting
Container mtusi-lab4-lb-1 Started
Container mtusi-lab4-s3-init-1 Waiting
Container mtusi-lab4-api-1 Waiting
Container mtusi-lab4-api-2 Waiting
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-s3-1 Waiting
Container mtusi-lab4-lb-1 Waiting
Container mtusi-lab4-valkey-1 Waiting
Container mtusi-lab4-migrate-1 Waiting
Container mtusi-lab4-valkey-1 Healthy
Container mtusi-lab4-migrate-1 Exited
Container mtusi-lab4-s3-init-1 Exited
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-s3-1 Healthy
Container mtusi-lab4-api-1 Healthy
Container mtusi-lab4-api-2 Healthy
Container mtusi-lab4-lb-1 Healthy
[exit 0, 31.3 s]
$ docker compose ps -a --format table {{.Service}} {{.Image}} {{.Status}}
SERVICE IMAGE STATUS
api mtusi-messenger:lab4 Up 12 seconds (healthy)
api mtusi-messenger:lab4 Up 12 seconds (healthy)
lb haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 Up 5 seconds (healthy)
migrate mtusi-messenger:lab4 Exited (0) 18 seconds ago
postgres postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c Up 25 seconds (healthy)
s3 chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 Up 25 seconds (healthy)
s3-init mtusi-messenger:lab4 Exited (0) 12 seconds ago
valkey valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 Up 25 seconds (healthy)
[exit 0, 0.3 s]
$ docker compose logs --no-log-prefix migrate
migrate: 0001_init: applied
migrate: 0002_idempotency_keys: applied
migrate: 0003_post_message_function: applied
migrate: 0004_attachments: applied
migrate: done, 4 migration file(s) checked
[exit 0, 0.3 s]
$ docker compose logs --no-log-prefix s3-init
s3-bootstrap: bucket messenger-attachments: created
s3-bootstrap: sentinel health/sentinel: written
s3-bootstrap: anonymous GET object: HTTP 403 -> refused
s3-bootstrap: anonymous LIST bucket: HTTP 403 -> refused
s3-bootstrap: api: read object: allowed -> OK
s3-bootstrap: api: write object: allowed -> OK
s3-bootstrap: api: delete object: allowed -> OK
s3-bootstrap: api: list bucket: refused (AccessDenied) -> OK
s3-bootstrap: api: create bucket: refused (AccessDenied) -> OK
s3-bootstrap: api: read object of another bucket: refused (403) -> OK
s3-bootstrap: reconciler: list bucket: allowed -> OK
s3-bootstrap: reconciler: create bucket: refused (AccessDenied) -> OK
s3-bootstrap: api: buckets visible in ListBuckets: [] -> OK
s3-bootstrap: done: private bucket, least-privilege keys verified
[exit 0, 0.3 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py replicas
mtusi-lab4-api-1 X-Instance-Id=8691c30aa702 running health=healthy pid=28760 started=2026-10-25T11:40:06.635 ip=172.25.0.2,172.26.0.3,172.27.0.4
mtusi-lab4-api-2 X-Instance-Id=4026e296947c running health=healthy pid=28857 started=2026-10-25T11:40:07.085 ip=172.25.0.3,172.26.0.4,172.27.0.5
[exit 0, 0.5 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api1 172.25.0.2 UP mtusi-lab4-api-1 X-Instance-Id=8691c30aa702
api2 172.25.0.3 UP mtusi-lab4-api-2 X-Instance-Id=4026e296947c
(other slots: MAINT, no container)
[exit 0, 0.7 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe -m venv .venv
[exit 0, 6.9 s]
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install -q --require-hashes --no-deps -r requirements.txt
[notice] A new release of pip is available: 26.1.2 -> 26.2.2
[notice] To update, run: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install --upgrade pip
[exit 0, 21.2 s]
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install -q -r requirements-test.txt -r requirements-dev.txt
[notice] A new release of pip is available: 26.1.2 -> 26.2.2
[notice] To update, run: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe -m pip install --upgrade pip
[exit 0, 14.0 s]
+16
View File
@@ -0,0 +1,16 @@
# Public checks repeated on the final commit fc50362 after the fixes (2026-10-25T12:34:34Z)
$ make test
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
$ make test-persistence ACTION_SCRIPT=scripts/restart.cmd
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
2 API replica(s) UP in HAProxy after 0.7 s: ['0e899ebdfc56', '92fdfb319248']
$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 5.837}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
mtusi-lab4-api-2 X-Instance-Id=0e899ebdfc56 exited health=unhealthy pid=0 started=2026-10-25T12:35:22.631 ip=-
2 API replica(s) UP in HAProxy after 0.6 s: ['0e899ebdfc56', '92fdfb319248']
$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 5.922}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
mtusi-lab4-api-1 X-Instance-Id=92fdfb319248 exited health=unhealthy pid=0 started=2026-10-25T12:35:22.981 ip=-
2 API replica(s) UP in HAProxy after 0.6 s: ['0e899ebdfc56', '92fdfb319248']
+328
View File
@@ -0,0 +1,328 @@
# Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py
# 2026-10-25T12:03:36Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/upload_memory.py --output evidence/memory
generating files ...
{"scenario": "png-9.4MiB x16, slots=4 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 16, "s3_sampled_max_mib": 548.3, "s3_anon_after_mib": 386.7, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 16}, "error_reasons": {}, "wall_seconds": 4.37, "latency_s": {"p50": 3.62, "max": 4.34}, "replicas": {"f2c108a7e1b2": {"idle_mib": 75.3, "peak_mib": 156.3, "sampled_max_mib": 156.1, "python_vmhwm_mib": 152.6, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 81.0}, "2e1bacf102e6": {"idle_mib": 74.5, "peak_mib": 163.2, "sampled_max_mib": 161.7, "python_vmhwm_mib": 152.9, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 88.7}}}
{"scenario": "png-9.4MiB x32, slots=4 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 32, "s3_sampled_max_mib": 800.5, "s3_anon_after_mib": 399.1, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 25, "503": 7}, "error_reasons": {"Слишком много одновременных загрузок, повторите позже": 7}, "wall_seconds": 6.96, "latency_s": {"p50": 4.99, "max": 6.91}, "replicas": {"5e7cbb39228d": {"idle_mib": 74.9, "peak_mib": 155.1, "sampled_max_mib": 153.6, "python_vmhwm_mib": 143.8, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 80.2}, "0036ead083c5": {"idle_mib": 74.6, "peak_mib": 187.5, "sampled_max_mib": 185.9, "python_vmhwm_mib": 181.2, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 112.9}}}
{"scenario": "png-9.4MiB x32, slots=16 (bound raised)", "settings": {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 32, "s3_sampled_max_mib": 1023.9, "s3_anon_after_mib": 411.8, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 24, "503": 8}, "error_reasons": {"Хранилище файлов временно недоступно": 8}, "wall_seconds": 10.2, "latency_s": {"p50": 7.46, "max": 10.04}, "replicas": {"04b50b0bdc7f": {"idle_mib": 75.2, "peak_mib": 284.6, "sampled_max_mib": 284.3, "python_vmhwm_mib": 285.9, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 209.4}, "f860d892caaf": {"idle_mib": 75.1, "peak_mib": 268.2, "sampled_max_mib": 265.8, "python_vmhwm_mib": 266.7, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 193.1}}}
{"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=1 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 16, "s3_sampled_max_mib": 990.1, "s3_anon_after_mib": 386.0, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 0.6, "types": ["image/jpeg"], "status_codes": {"201": 16}, "error_reasons": {}, "wall_seconds": 5.77, "latency_s": {"p50": 3.58, "max": 5.74}, "replicas": {"2c78d3ce98f5": {"idle_mib": 75.0, "peak_mib": 317.1, "sampled_max_mib": 315.5, "python_vmhwm_mib": 107.5, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 242.1}, "2220bc508f5c": {"idle_mib": 74.7, "peak_mib": 316.5, "sampled_max_mib": 315.5, "python_vmhwm_mib": 107.2, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 241.8}}}
{"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=4", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, "clients": 16, "s3_sampled_max_mib": 995.3, "s3_anon_after_mib": 386.6, "s3_limit_mib": 1024.0, "incidents": {"mtusi-lab4-api-1": "restarts=0 oom_kill=6", "mtusi-lab4-api-2": "restarts=0 oom_kill=7"}, "file_mib_each": 0.6, "types": ["image/jpeg"], "status_codes": {"201": 2, "503": 14}, "error_reasons": {"Не хватило ресурсов для проверки изображения, повторите позже": 13, "Слишком много одновременных загрузок, повторите позже": 1}, "wall_seconds": 8.15, "latency_s": {"p50": 8.12, "max": 8.12}, "replicas": {"984d6a81b4b8": {"idle_mib": 86.2, "peak_mib": 512.0, "sampled_max_mib": 511.9, "python_vmhwm_mib": 110.7, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 425.8}, "42f112fb4559": {"idle_mib": 74.5, "peak_mib": 512.0, "sampled_max_mib": 507.9, "python_vmhwm_mib": 92.3, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 437.5}}}
{"date": "2026-10-25 15:08:28", "host": "Windows-10-10.0.19045-SP0", "cpus": 8, "commit": "f9f5ab9", "api_limits": "1000000000 536870912", "jpeg_file_mib": 0.6, "png_file_mib": 9.42}
[exit 0, 292.6 s]
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/reconcile_faults.py --output evidence/reconcile
{
"dry_run_changed_nothing": true,
"per_key": {
"dry-run": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": {
"kind": "missing_object",
"result": "planned",
"used": true
},
"unused, object truncated D": {
"kind": "size_mismatch",
"result": "planned",
"used": false
},
"quarantined but healthy E": {
"kind": "restored_object",
"result": "planned",
"used": true
},
"same-size corruption F": {
"kind": "checksum_mismatch",
"result": "planned",
"used": false
},
"old orphan object": {
"kind": "orphan_object",
"result": "planned",
"used": false
},
"stale intent + object": {
"kind": "stale_intent",
"result": "planned",
"used": false
},
"stale intent, no object": {
"kind": "stale_intent",
"result": "planned",
"used": false
},
"fresh orphan (in-flight)": {
"kind": "fresh_orphan_object",
"result": "not_actionable",
"used": false
},
"fresh intent (in-flight)": {
"kind": "fresh_intent",
"result": "not_actionable",
"used": false
}
},
"apply": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": {
"kind": "missing_object",
"result": "applied",
"used": true
},
"unused, object truncated D": {
"kind": "size_mismatch",
"result": "applied",
"used": false
},
"quarantined but healthy E": {
"kind": "restored_object",
"result": "applied",
"used": true
},
"same-size corruption F": {
"kind": "checksum_mismatch",
"result": "applied",
"used": false
},
"old orphan object": {
"kind": "orphan_object",
"result": "applied",
"used": false
},
"stale intent + object": {
"kind": "stale_intent",
"result": "applied",
"used": false
},
"stale intent, no object": {
"kind": "stale_intent",
"result": "applied",
"used": false
},
"fresh orphan (in-flight)": {
"kind": "fresh_orphan_object",
"result": "not_actionable",
"used": false
},
"fresh intent (in-flight)": {
"kind": "fresh_intent",
"result": "not_actionable",
"used": false
}
},
"apply again": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": null,
"unused, object truncated D": null,
"quarantined but healthy E": null,
"same-size corruption F": null,
"old orphan object": null,
"stale intent + object": null,
"stale intent, no object": null,
"fresh orphan (in-flight)": {
"kind": "fresh_orphan_object",
"result": "not_actionable",
"used": false
},
"fresh intent (in-flight)": {
"kind": "fresh_intent",
"result": "not_actionable",
"used": false
}
},
"after restore of C": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": {
"kind": "restored_object",
"result": "applied",
"used": true
},
"unused, object truncated D": null,
"quarantined but healthy E": null,
"same-size corruption F": null,
"old orphan object": null,
"stale intent + object": null,
"stale intent, no object": null,
"fresh orphan (in-flight)": {
"kind": "orphan_object",
"result": "applied",
"used": false
},
"fresh intent (in-flight)": {
"kind": "stale_intent",
"result": "applied",
"used": false
}
}
},
"objects_present_at_end (fresh items were older than --grace by the later runs)": {
"healthy used A": true,
"healthy unused B": true,
"used, object deleted C": true,
"unused, object truncated D": true,
"quarantined but healthy E": true,
"same-size corruption F": true,
"old orphan object": false,
"stale intent + object": false,
"stale intent, no object": false,
"fresh orphan (in-flight)": false,
"fresh intent (in-flight)": false
},
"rows_after": {
"C": {
"status": "ready",
"lost_reason": null
},
"D": {
"status": "ready",
"lost_reason": "size_mismatch"
},
"E": {
"status": "ready",
"lost_reason": null
},
"F": {
"status": "ready",
"lost_reason": "checksum_mismatch"
},
"A": {
"status": "ready",
"lost_reason": null
},
"B": {
"status": "ready",
"lost_reason": null
}
},
"content_status_after": {
"A": 200,
"B": 200,
"C": 200,
"D": 404,
"E": 200,
"F": 404
},
"durations_s": [
3.94,
3.983,
3.0,
3.548
]
}
{
"orphans_injected": 2000,
"interrupted_run": {
"exit_code": 1,
"error": "store_unavailable: Unavailable",
"applied": 440
},
"orphans_left_after_interrupt": 1560,
"rerun": {
"exit_code": 0,
"applied": 1562,
"error": null
},
"third": {
"exit_code": 0,
"planned": 0
},
"orphans_left_at_end": 0
}
[exit 0, 182.1 s]
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/reconcile_cost.py --output evidence/reconcile
removed 0 synthetic attachments
Traceback (most recent call last):
File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 155, in <module>
sys.exit(main())
~~~~^^
File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 127, in main
add(target - present, owner, chat)
~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 89, in add
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
"object_key, status, created_at, updated_at) "
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 62, in psql
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
"ON_ERROR_STOP=1", "-c", sql).stdout.strip()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 57, in compose
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
encoding="utf-8", timeout=timeout, check=check)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 555, in run
with Popen(*popenargs, **kwargs) as process:
~~~~~^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 1039, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pass_fds, cwd, env,
^^^^^^^^^^^^^^^^^^^
...<5 lines>...
gid, gids, uid, umask,
^^^^^^^^^^^^^^^^^^^^^^
start_new_session, process_group)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 1553, in _execute_child
hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
# no special security
^^^^^^^^^^^^^^^^^^^^^
...<4 lines>...
cwd,
^^^^
startupinfo)
^^^^^^^^^^^^
FileNotFoundError: [WinError 206] Имя файла или его расширение имеет слишком большую длину
[exit 1, 1.3 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['379d13a0775a', '862067d68c1c']
[exit 0, 1.4 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
{
"lab": 4,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.091,
"concurrency": 4,
"successful_requests": 5416,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 179.989,
"successful_latency_ms": {
"p50": 17.684,
"p95": 49.053,
"p99": 63.81
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
[exit 0, 30.7 s]
$ docker compose ps -a --format table {{.Service}} {{.Status}}
SERVICE STATUS
api Up 3 minutes (healthy)
api Up 3 minutes (healthy)
lb Up 24 minutes (healthy)
migrate Exited (0) 10 minutes ago
postgres Up 11 minutes (healthy)
s3 Up 6 minutes (healthy)
s3-init Exited (0) 10 minutes ago
valkey Up 12 minutes (healthy)
[exit 0, 0.4 s]
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 4,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.091,
"concurrency": 4,
"successful_requests": 5416,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 179.989,
"successful_latency_ms": {
"p50": 17.684,
"p95": 49.053,
"p99": 63.81
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
# make check, make validate (venv), make test
# 2026-10-25T11:41:03Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов
$ make check
python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
[exit 0, 1.3 s]
$ make validate PYTHON=.venv/Scripts/python.exe
.venv/Scripts/python.exe scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
[exit 0, 2.3 s]
$ make test
python tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
[exit 0, 1.3 s]
+251
View File
@@ -0,0 +1,251 @@
{
"meta": {
"date": "2026-10-25 15:08:28",
"host": "Windows-10-10.0.19045-SP0",
"cpus": 8,
"commit": "f9f5ab9",
"api_limits": "1000000000 536870912",
"jpeg_file_mib": 0.6,
"png_file_mib": 9.42
},
"scenarios": [
{
"scenario": "png-9.4MiB x16, slots=4 (default)",
"settings": {
"UPLOAD_CONCURRENCY": "4",
"IMAGE_VERIFY_CONCURRENCY": "1"
},
"clients": 16,
"s3_sampled_max_mib": 548.3,
"s3_anon_after_mib": 386.7,
"s3_limit_mib": 1024.0,
"incidents": {},
"file_mib_each": 9.42,
"types": [
"image/png"
],
"status_codes": {
"201": 16
},
"error_reasons": {},
"wall_seconds": 4.37,
"latency_s": {
"p50": 3.62,
"max": 4.34
},
"replicas": {
"f2c108a7e1b2": {
"idle_mib": 75.3,
"peak_mib": 156.3,
"sampled_max_mib": 156.1,
"python_vmhwm_mib": 152.6,
"uploads_served": 8,
"limit_mib": 512.0,
"peak_minus_idle_mib": 81.0
},
"2e1bacf102e6": {
"idle_mib": 74.5,
"peak_mib": 163.2,
"sampled_max_mib": 161.7,
"python_vmhwm_mib": 152.9,
"uploads_served": 8,
"limit_mib": 512.0,
"peak_minus_idle_mib": 88.7
}
}
},
{
"scenario": "png-9.4MiB x32, slots=4 (default)",
"settings": {
"UPLOAD_CONCURRENCY": "4",
"IMAGE_VERIFY_CONCURRENCY": "1"
},
"clients": 32,
"s3_sampled_max_mib": 800.5,
"s3_anon_after_mib": 399.1,
"s3_limit_mib": 1024.0,
"incidents": {},
"file_mib_each": 9.42,
"types": [
"image/png"
],
"status_codes": {
"201": 25,
"503": 7
},
"error_reasons": {
"Слишком много одновременных загрузок, повторите позже": 7
},
"wall_seconds": 6.96,
"latency_s": {
"p50": 4.99,
"max": 6.91
},
"replicas": {
"5e7cbb39228d": {
"idle_mib": 74.9,
"peak_mib": 155.1,
"sampled_max_mib": 153.6,
"python_vmhwm_mib": 143.8,
"uploads_served": 16,
"limit_mib": 512.0,
"peak_minus_idle_mib": 80.2
},
"0036ead083c5": {
"idle_mib": 74.6,
"peak_mib": 187.5,
"sampled_max_mib": 185.9,
"python_vmhwm_mib": 181.2,
"uploads_served": 16,
"limit_mib": 512.0,
"peak_minus_idle_mib": 112.9
}
}
},
{
"scenario": "png-9.4MiB x32, slots=16 (bound raised)",
"settings": {
"UPLOAD_CONCURRENCY": "16",
"IMAGE_VERIFY_CONCURRENCY": "1"
},
"clients": 32,
"s3_sampled_max_mib": 1023.9,
"s3_anon_after_mib": 411.8,
"s3_limit_mib": 1024.0,
"incidents": {},
"file_mib_each": 9.42,
"types": [
"image/png"
],
"status_codes": {
"201": 24,
"503": 8
},
"error_reasons": {
"Хранилище файлов временно недоступно": 8
},
"wall_seconds": 10.2,
"latency_s": {
"p50": 7.46,
"max": 10.04
},
"replicas": {
"04b50b0bdc7f": {
"idle_mib": 75.2,
"peak_mib": 284.6,
"sampled_max_mib": 284.3,
"python_vmhwm_mib": 285.9,
"uploads_served": 16,
"limit_mib": 512.0,
"peak_minus_idle_mib": 209.4
},
"f860d892caaf": {
"idle_mib": 75.1,
"peak_mib": 268.2,
"sampled_max_mib": 265.8,
"python_vmhwm_mib": 266.7,
"uploads_served": 16,
"limit_mib": 512.0,
"peak_minus_idle_mib": 193.1
}
}
},
{
"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=1 (default)",
"settings": {
"UPLOAD_CONCURRENCY": "4",
"IMAGE_VERIFY_CONCURRENCY": "1"
},
"clients": 16,
"s3_sampled_max_mib": 990.1,
"s3_anon_after_mib": 386.0,
"s3_limit_mib": 1024.0,
"incidents": {},
"file_mib_each": 0.6,
"types": [
"image/jpeg"
],
"status_codes": {
"201": 16
},
"error_reasons": {},
"wall_seconds": 5.77,
"latency_s": {
"p50": 3.58,
"max": 5.74
},
"replicas": {
"2c78d3ce98f5": {
"idle_mib": 75.0,
"peak_mib": 317.1,
"sampled_max_mib": 315.5,
"python_vmhwm_mib": 107.5,
"uploads_served": 8,
"limit_mib": 512.0,
"peak_minus_idle_mib": 242.1
},
"2220bc508f5c": {
"idle_mib": 74.7,
"peak_mib": 316.5,
"sampled_max_mib": 315.5,
"python_vmhwm_mib": 107.2,
"uploads_served": 8,
"limit_mib": 512.0,
"peak_minus_idle_mib": 241.8
}
}
},
{
"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=4",
"settings": {
"UPLOAD_CONCURRENCY": "4",
"IMAGE_VERIFY_CONCURRENCY": "4"
},
"clients": 16,
"s3_sampled_max_mib": 995.3,
"s3_anon_after_mib": 386.6,
"s3_limit_mib": 1024.0,
"incidents": {
"mtusi-lab4-api-1": "restarts=0 oom_kill=6",
"mtusi-lab4-api-2": "restarts=0 oom_kill=7"
},
"file_mib_each": 0.6,
"types": [
"image/jpeg"
],
"status_codes": {
"201": 2,
"503": 14
},
"error_reasons": {
"Не хватило ресурсов для проверки изображения, повторите позже": 13,
"Слишком много одновременных загрузок, повторите позже": 1
},
"wall_seconds": 8.15,
"latency_s": {
"p50": 8.12,
"max": 8.12
},
"replicas": {
"984d6a81b4b8": {
"idle_mib": 86.2,
"peak_mib": 512.0,
"sampled_max_mib": 511.9,
"python_vmhwm_mib": 110.7,
"uploads_served": 8,
"limit_mib": 512.0,
"peak_minus_idle_mib": 425.8
},
"42f112fb4559": {
"idle_mib": 74.5,
"peak_mib": 512.0,
"sampled_max_mib": 507.9,
"python_vmhwm_mib": 92.3,
"uploads_served": 8,
"limit_mib": 512.0,
"peak_minus_idle_mib": 437.5
}
}
}
]
}
+14
View File
@@ -0,0 +1,14 @@
# Reconciler fix 6fb70de (checksum quarantine lifted only with --verify-checksums), 2026-10-25T12:47:16Z
$ docker compose run --rm reconciler --apply (plain pass right after quarantining 3 leftover rows with --verify-checksums)
findings {'fresh_intent': 2} actions {'planned': 0, 'applied': 0, 'skipped': 0, 'failed': 0}
$ python -m pytest student-tests/test_reconcile.py -v
student-tests/test_reconcile.py::test_detect_and_safely_fix_faults_on_both_sides PASSED [ 16%]
student-tests/test_reconcile.py::test_checksum_mode_finds_same_size_corruption PASSED [ 33%]
student-tests/test_reconcile.py::test_safety_valve_refuses_mass_deletion PASSED [ 50%]
student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[s3] PASSED [ 66%]
student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[postgres] PASSED [ 83%]
student-tests/test_reconcile.py::test_live_uploads_are_never_hurt_by_concurrent_apply PASSED [100%]
======================== 6 passed in 386.80s (0:06:26) ========================
# make test on 6fb70de (2026-10-25T12:54:03Z)
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+139
View File
@@ -0,0 +1,139 @@
{
"meta": {
"date": "2026-10-25 15:45:05",
"host": "Windows-10-10.0.19045-SP0",
"cpus": 8,
"commit": "fc50362",
"object_bytes": 1024,
"reconciler_limits": "cpus 1.0, memory 256m (compose)",
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"
},
"results": [
{
"synthetic": 1000,
"label": "dry-run, page 1000",
"args": [],
"runs": 3,
"duration_s_median": 0.301,
"duration_s_all": [
0.353,
0.289,
0.301
],
"objects": 1285,
"rows": 1286,
"findings": {
"fresh_intent": 2
},
"cost": {
"s3_list_requests": 2,
"s3_head_requests": 0,
"s3_get_bytes": 0,
"db_pages": 2,
"page_size": 1000
}
},
{
"synthetic": 10000,
"label": "dry-run, page 1000",
"args": [],
"runs": 3,
"duration_s_median": 1.669,
"duration_s_all": [
1.669,
1.744,
1.653
],
"objects": 10285,
"rows": 10286,
"findings": {
"fresh_intent": 2
},
"cost": {
"s3_list_requests": 11,
"s3_head_requests": 0,
"s3_get_bytes": 0,
"db_pages": 11,
"page_size": 1000
}
},
{
"synthetic": 10000,
"label": "dry-run, page 100",
"args": [
"--page-size",
"100"
],
"runs": 3,
"duration_s_median": 2.278,
"duration_s_all": [
2.278,
1.999,
2.312
],
"objects": 10285,
"rows": 10286,
"findings": {
"fresh_intent": 2
},
"cost": {
"s3_list_requests": 103,
"s3_head_requests": 0,
"s3_get_bytes": 0,
"db_pages": 103,
"page_size": 100
}
},
{
"synthetic": 10000,
"label": "dry-run, --verify-checksums",
"args": [
"--verify-checksums"
],
"runs": 3,
"duration_s_median": 48.411,
"duration_s_all": [
53.483,
48.411,
48.288
],
"objects": 10285,
"rows": 10286,
"findings": {
"checksum_mismatch": 3,
"fresh_intent": 2
},
"cost": {
"s3_list_requests": 11,
"s3_head_requests": 0,
"s3_get_bytes": 694445075,
"db_pages": 11,
"page_size": 1000
}
},
{
"synthetic": 50000,
"label": "dry-run, page 1000",
"args": [],
"runs": 3,
"duration_s_median": 8.435,
"duration_s_all": [
8.538,
8.435,
8.139
],
"objects": 50285,
"rows": 50286,
"findings": {
"fresh_intent": 2
},
"cost": {
"s3_list_requests": 51,
"s3_head_requests": 0,
"s3_get_bytes": 0,
"db_pages": 51,
"page_size": 1000
}
}
]
}
+188
View File
@@ -0,0 +1,188 @@
{
"mode": "dry-run",
"grace_seconds": 30.0,
"verify_checksums": true,
"duration_seconds": 3.94,
"error": null,
"scanned": {
"objects": 198,
"rows": 199,
"consistent": 181
},
"findings": {
"orphan_object": 1,
"stale_intent": 12,
"missing_object": 1,
"size_mismatch": 1,
"checksum_mismatch": 2,
"restored_object": 1,
"fresh_orphan_object": 1,
"fresh_intent": 1,
"fresh_missing_object": 0
},
"used_attachments_in_findings": 2,
"actions": {
"planned": 18,
"applied": 0,
"skipped": 0,
"failed": 0
},
"cost": {
"s3_list_requests": 1,
"s3_head_requests": 0,
"s3_get_bytes": 668556368,
"db_pages": 1,
"page_size": 1000
},
"samples": {
"stale_intent": [
{
"key": "o/02e47f043e4a2d0fdfb8823308b5a6f8",
"attachment_id": "445129ca-5e0a-48eb-9e9a-cb841aa3c21f",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/3738b895df9e4a2af8e83d5c31cbdb97",
"attachment_id": "418dcc4d-a2e9-4653-8209-5a8b77f6cbba",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/3f04e9b122374a366667ec50283ff028",
"attachment_id": "7832128b-d4b0-4128-92d0-fa4d9e998789",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/58646a726a50106dfeb7b214a0b68e45",
"attachment_id": "8919db52-4396-46c5-99cd-13df371436fc",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/5ca75dbe045e57c0683a60a2da3b4801",
"attachment_id": "53062ac4-cfbb-4393-944c-a2931ca2e34d",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/74c65b3fa98dc631e6f57909079c4da3",
"attachment_id": "a71a1c20-f815-49a7-945d-e45bdd4afec3",
"used": false,
"result": "planned",
"detail": "object absent"
},
{
"key": "o/b65a274dcfc743b9b07ac223e9846546",
"attachment_id": "723cb122-1fd3-41e5-8e7f-712e39066ae3",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/c412437db6184a939bed9283b11a94f4",
"attachment_id": "58548b0c-4025-47c7-8377-28b85bfc1d46",
"used": false,
"result": "planned",
"detail": "object absent"
},
{
"key": "o/c5326823b48696d02dd38321e5ddedde",
"attachment_id": "cbe8966a-bb9e-49f5-8799-fdb341929f15",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/cf65354466733e9171737583f869c0fc",
"attachment_id": "71b2e4a6-615e-4d6c-a35c-34ac0f703662",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/d3dca5b6df5b097cfd537a6b3ffd3deb",
"attachment_id": "ab9acee5-a57a-4698-8724-cd08431fa502",
"used": false,
"result": "planned",
"detail": "object present"
},
{
"key": "o/e8cda21ebf7a465d6687da4c0b3c7a2a",
"attachment_id": "99d11e03-ae13-4255-b9a9-8ee8679024ff",
"used": false,
"result": "planned",
"detail": "object present"
}
],
"fresh_orphan_object": [
{
"key": "o/04232d5e57374a09b5e5368916176439",
"attachment_id": null,
"used": false,
"result": "not_actionable"
}
],
"fresh_intent": [
{
"key": "o/2a286f02c3854017886a0b371d08f2c9",
"attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6",
"used": false,
"result": "not_actionable",
"detail": "object present"
}
],
"size_mismatch": [
{
"key": "o/5be370c6443c51303e893c28529376b6",
"attachment_id": "a65a12d5-9369-4322-93ce-62f8f8a142a0",
"used": false,
"result": "planned"
}
],
"checksum_mismatch": [
{
"key": "o/77f4fbc630a1966cb3f6d4e09c019318",
"attachment_id": "ed0ff4c7-507b-44cd-8928-622ea6e977a8",
"used": false,
"result": "planned"
},
{
"key": "o/ee7f3d21bebb4a56572b9b1f7d2299ee",
"attachment_id": "4c72dcfe-c46f-47a0-a442-2937c722ed99",
"used": false,
"result": "planned"
}
],
"missing_object": [
{
"key": "o/ac353be8ed77f5c230f7be71809ce405",
"attachment_id": "89e1b8ca-e554-4691-92b0-667b46757d10",
"used": true,
"result": "planned"
}
],
"restored_object": [
{
"key": "o/adc1d649f5b94260bd28445fb55ac5fa",
"attachment_id": "becdb821-076e-4bff-ada3-d0bf6823cce7",
"used": true,
"result": "planned"
}
],
"orphan_object": [
{
"key": "o/d2b0819448ab4d06a1dca06df4fac200",
"attachment_id": null,
"used": false,
"result": "planned"
}
]
}
}
+188
View File
@@ -0,0 +1,188 @@
{
"mode": "apply",
"grace_seconds": 30.0,
"verify_checksums": true,
"duration_seconds": 3.983,
"error": null,
"scanned": {
"objects": 198,
"rows": 199,
"consistent": 181
},
"findings": {
"orphan_object": 1,
"stale_intent": 12,
"missing_object": 1,
"size_mismatch": 1,
"checksum_mismatch": 2,
"restored_object": 1,
"fresh_orphan_object": 1,
"fresh_intent": 1,
"fresh_missing_object": 0
},
"used_attachments_in_findings": 2,
"actions": {
"planned": 18,
"applied": 18,
"skipped": 0,
"failed": 0
},
"cost": {
"s3_list_requests": 1,
"s3_head_requests": 5,
"s3_get_bytes": 668676391,
"db_pages": 1,
"page_size": 1000
},
"samples": {
"stale_intent": [
{
"key": "o/02e47f043e4a2d0fdfb8823308b5a6f8",
"attachment_id": "445129ca-5e0a-48eb-9e9a-cb841aa3c21f",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/3738b895df9e4a2af8e83d5c31cbdb97",
"attachment_id": "418dcc4d-a2e9-4653-8209-5a8b77f6cbba",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/3f04e9b122374a366667ec50283ff028",
"attachment_id": "7832128b-d4b0-4128-92d0-fa4d9e998789",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/58646a726a50106dfeb7b214a0b68e45",
"attachment_id": "8919db52-4396-46c5-99cd-13df371436fc",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/5ca75dbe045e57c0683a60a2da3b4801",
"attachment_id": "53062ac4-cfbb-4393-944c-a2931ca2e34d",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/74c65b3fa98dc631e6f57909079c4da3",
"attachment_id": "a71a1c20-f815-49a7-945d-e45bdd4afec3",
"used": false,
"result": "applied",
"detail": "object absent"
},
{
"key": "o/b65a274dcfc743b9b07ac223e9846546",
"attachment_id": "723cb122-1fd3-41e5-8e7f-712e39066ae3",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/c412437db6184a939bed9283b11a94f4",
"attachment_id": "58548b0c-4025-47c7-8377-28b85bfc1d46",
"used": false,
"result": "applied",
"detail": "object absent"
},
{
"key": "o/c5326823b48696d02dd38321e5ddedde",
"attachment_id": "cbe8966a-bb9e-49f5-8799-fdb341929f15",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/cf65354466733e9171737583f869c0fc",
"attachment_id": "71b2e4a6-615e-4d6c-a35c-34ac0f703662",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/d3dca5b6df5b097cfd537a6b3ffd3deb",
"attachment_id": "ab9acee5-a57a-4698-8724-cd08431fa502",
"used": false,
"result": "applied",
"detail": "object present"
},
{
"key": "o/e8cda21ebf7a465d6687da4c0b3c7a2a",
"attachment_id": "99d11e03-ae13-4255-b9a9-8ee8679024ff",
"used": false,
"result": "applied",
"detail": "object present"
}
],
"fresh_orphan_object": [
{
"key": "o/04232d5e57374a09b5e5368916176439",
"attachment_id": null,
"used": false,
"result": "not_actionable"
}
],
"fresh_intent": [
{
"key": "o/2a286f02c3854017886a0b371d08f2c9",
"attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6",
"used": false,
"result": "not_actionable",
"detail": "object present"
}
],
"size_mismatch": [
{
"key": "o/5be370c6443c51303e893c28529376b6",
"attachment_id": "a65a12d5-9369-4322-93ce-62f8f8a142a0",
"used": false,
"result": "applied"
}
],
"checksum_mismatch": [
{
"key": "o/77f4fbc630a1966cb3f6d4e09c019318",
"attachment_id": "ed0ff4c7-507b-44cd-8928-622ea6e977a8",
"used": false,
"result": "applied"
},
{
"key": "o/ee7f3d21bebb4a56572b9b1f7d2299ee",
"attachment_id": "4c72dcfe-c46f-47a0-a442-2937c722ed99",
"used": false,
"result": "applied"
}
],
"missing_object": [
{
"key": "o/ac353be8ed77f5c230f7be71809ce405",
"attachment_id": "89e1b8ca-e554-4691-92b0-667b46757d10",
"used": true,
"result": "applied"
}
],
"restored_object": [
{
"key": "o/adc1d649f5b94260bd28445fb55ac5fa",
"attachment_id": "becdb821-076e-4bff-ada3-d0bf6823cce7",
"used": true,
"result": "applied"
}
],
"orphan_object": [
{
"key": "o/d2b0819448ab4d06a1dca06df4fac200",
"attachment_id": null,
"used": false,
"result": "applied"
}
]
}
}
@@ -0,0 +1,56 @@
{
"mode": "apply",
"grace_seconds": 30.0,
"verify_checksums": true,
"duration_seconds": 3.0,
"error": null,
"scanned": {
"objects": 187,
"rows": 187,
"consistent": 186
},
"findings": {
"orphan_object": 0,
"stale_intent": 0,
"missing_object": 0,
"size_mismatch": 0,
"checksum_mismatch": 0,
"restored_object": 0,
"fresh_orphan_object": 1,
"fresh_intent": 1,
"fresh_missing_object": 0
},
"used_attachments_in_findings": 0,
"actions": {
"planned": 0,
"applied": 0,
"skipped": 0,
"failed": 0
},
"cost": {
"s3_list_requests": 1,
"s3_head_requests": 0,
"s3_get_bytes": 668556368,
"db_pages": 1,
"page_size": 1000
},
"samples": {
"fresh_orphan_object": [
{
"key": "o/04232d5e57374a09b5e5368916176439",
"attachment_id": null,
"used": false,
"result": "not_actionable"
}
],
"fresh_intent": [
{
"key": "o/2a286f02c3854017886a0b371d08f2c9",
"attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6",
"used": false,
"result": "not_actionable",
"detail": "object present"
}
]
}
}
@@ -0,0 +1,64 @@
{
"mode": "apply",
"grace_seconds": 30.0,
"verify_checksums": true,
"duration_seconds": 3.548,
"error": null,
"scanned": {
"objects": 188,
"rows": 187,
"consistent": 185
},
"findings": {
"orphan_object": 1,
"stale_intent": 1,
"missing_object": 0,
"size_mismatch": 0,
"checksum_mismatch": 0,
"restored_object": 1,
"fresh_orphan_object": 0,
"fresh_intent": 0,
"fresh_missing_object": 0
},
"used_attachments_in_findings": 1,
"actions": {
"planned": 3,
"applied": 3,
"skipped": 0,
"failed": 0
},
"cost": {
"s3_list_requests": 1,
"s3_head_requests": 2,
"s3_get_bytes": 668564124,
"db_pages": 1,
"page_size": 1000
},
"samples": {
"orphan_object": [
{
"key": "o/04232d5e57374a09b5e5368916176439",
"attachment_id": null,
"used": false,
"result": "applied"
}
],
"stale_intent": [
{
"key": "o/2a286f02c3854017886a0b371d08f2c9",
"attachment_id": "697c1bdb-dee1-4a43-b26e-131a02cba9c6",
"used": false,
"result": "applied",
"detail": "object present"
}
],
"restored_object": [
{
"key": "o/ac353be8ed77f5c230f7be71809ce405",
"attachment_id": "89e1b8ca-e554-4691-92b0-667b46757d10",
"used": true,
"result": "applied"
}
]
}
}
+226
View File
@@ -0,0 +1,226 @@
{
"date": "2026-10-25 15:11:30",
"commit": "f9f5ab9",
"grace_seconds": 30,
"scenario_a": {
"dry_run_changed_nothing": true,
"per_key": {
"dry-run": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": {
"kind": "missing_object",
"result": "planned",
"used": true
},
"unused, object truncated D": {
"kind": "size_mismatch",
"result": "planned",
"used": false
},
"quarantined but healthy E": {
"kind": "restored_object",
"result": "planned",
"used": true
},
"same-size corruption F": {
"kind": "checksum_mismatch",
"result": "planned",
"used": false
},
"old orphan object": {
"kind": "orphan_object",
"result": "planned",
"used": false
},
"stale intent + object": {
"kind": "stale_intent",
"result": "planned",
"used": false
},
"stale intent, no object": {
"kind": "stale_intent",
"result": "planned",
"used": false
},
"fresh orphan (in-flight)": {
"kind": "fresh_orphan_object",
"result": "not_actionable",
"used": false
},
"fresh intent (in-flight)": {
"kind": "fresh_intent",
"result": "not_actionable",
"used": false
}
},
"apply": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": {
"kind": "missing_object",
"result": "applied",
"used": true
},
"unused, object truncated D": {
"kind": "size_mismatch",
"result": "applied",
"used": false
},
"quarantined but healthy E": {
"kind": "restored_object",
"result": "applied",
"used": true
},
"same-size corruption F": {
"kind": "checksum_mismatch",
"result": "applied",
"used": false
},
"old orphan object": {
"kind": "orphan_object",
"result": "applied",
"used": false
},
"stale intent + object": {
"kind": "stale_intent",
"result": "applied",
"used": false
},
"stale intent, no object": {
"kind": "stale_intent",
"result": "applied",
"used": false
},
"fresh orphan (in-flight)": {
"kind": "fresh_orphan_object",
"result": "not_actionable",
"used": false
},
"fresh intent (in-flight)": {
"kind": "fresh_intent",
"result": "not_actionable",
"used": false
}
},
"apply again": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": null,
"unused, object truncated D": null,
"quarantined but healthy E": null,
"same-size corruption F": null,
"old orphan object": null,
"stale intent + object": null,
"stale intent, no object": null,
"fresh orphan (in-flight)": {
"kind": "fresh_orphan_object",
"result": "not_actionable",
"used": false
},
"fresh intent (in-flight)": {
"kind": "fresh_intent",
"result": "not_actionable",
"used": false
}
},
"after restore of C": {
"healthy used A": null,
"healthy unused B": null,
"used, object deleted C": {
"kind": "restored_object",
"result": "applied",
"used": true
},
"unused, object truncated D": null,
"quarantined but healthy E": null,
"same-size corruption F": null,
"old orphan object": null,
"stale intent + object": null,
"stale intent, no object": null,
"fresh orphan (in-flight)": {
"kind": "orphan_object",
"result": "applied",
"used": false
},
"fresh intent (in-flight)": {
"kind": "stale_intent",
"result": "applied",
"used": false
}
}
},
"objects_present_at_end (fresh items were older than --grace by the later runs)": {
"healthy used A": true,
"healthy unused B": true,
"used, object deleted C": true,
"unused, object truncated D": true,
"quarantined but healthy E": true,
"same-size corruption F": true,
"old orphan object": false,
"stale intent + object": false,
"stale intent, no object": false,
"fresh orphan (in-flight)": false,
"fresh intent (in-flight)": false
},
"rows_after": {
"C": {
"status": "ready",
"lost_reason": null
},
"D": {
"status": "ready",
"lost_reason": "size_mismatch"
},
"E": {
"status": "ready",
"lost_reason": null
},
"F": {
"status": "ready",
"lost_reason": "checksum_mismatch"
},
"A": {
"status": "ready",
"lost_reason": null
},
"B": {
"status": "ready",
"lost_reason": null
}
},
"content_status_after": {
"A": 200,
"B": 200,
"C": 200,
"D": 404,
"E": 200,
"F": 404
},
"durations_s": [
3.94,
3.983,
3.0,
3.548
]
},
"scenario_b_interrupted": {
"orphans_injected": 2000,
"interrupted_run": {
"exit_code": 1,
"error": "store_unavailable: Unavailable",
"applied": 440
},
"orphans_left_after_interrupt": 1560,
"rerun": {
"exit_code": 0,
"applied": 1562,
"error": null
},
"third": {
"exit_code": 0,
"planned": 0
},
"orphans_left_at_end": 0
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,38 @@
{
"mode": "apply",
"grace_seconds": 30.0,
"verify_checksums": false,
"duration_seconds": 0.103,
"error": null,
"scanned": {
"objects": 186,
"rows": 186,
"consistent": 186
},
"findings": {
"orphan_object": 0,
"stale_intent": 0,
"missing_object": 0,
"size_mismatch": 0,
"checksum_mismatch": 0,
"restored_object": 0,
"fresh_orphan_object": 0,
"fresh_intent": 0,
"fresh_missing_object": 0
},
"used_attachments_in_findings": 0,
"actions": {
"planned": 0,
"applied": 0,
"skipped": 0,
"failed": 0
},
"cost": {
"s3_list_requests": 1,
"s3_head_requests": 0,
"s3_get_bytes": 0,
"db_pages": 1,
"page_size": 1000
},
"samples": {}
}
@@ -0,0 +1,414 @@
# Own tests: unit + stand (pytest -v); failures restored in finally
# 2026-10-25T11:43:04Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов
$ python -m pytest student-tests -v
============================= test session starts =============================
platform win32 -- Python 3.14.6, pytest-9.1.1, pluggy-1.7.0 -- C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe
rootdir: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean
plugins: anyio-4.15.1
collecting ... collected 320 items
student-tests/test_api.py::test_register_never_returns_password_and_login_returns_bearer PASSED [ 0%]
student-tests/test_api.py::test_duplicate_username_is_409 PASSED [ 0%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[short-11ch] PASSED [ 0%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[ ] PASSED [ 1%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[ \t \n \t \n \t \n ] PASSED [ 1%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 1%]
student-tests/test_api.py::test_edge_passwords_register_and_login[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 2%]
student-tests/test_api.py::test_edge_passwords_register_and_login[with:colons:inside] PASSED [ 2%]
student-tests/test_api.py::test_edge_passwords_register_and_login[Twelve chars] PASSED [ 2%]
student-tests/test_api.py::test_wrong_password_and_unknown_user_give_the_same_401 PASSED [ 3%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic \xe9] PASSED [ 3%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic !!!] PASSED [ 3%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic bm9fY29sb24=] PASSED [ 4%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic //46eA==] PASSED [ 4%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 4%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[] PASSED [ 5%]
student-tests/test_api.py::test_logout_revokes_only_this_token PASSED [ 5%]
student-tests/test_api.py::test_x_user_id_basic_and_garbage_do_not_authenticate PASSED [ 5%]
student-tests/test_api.py::test_malformed_json_is_400[{] PASSED [ 5%]
student-tests/test_api.py::test_malformed_json_is_400[] PASSED [ 6%]
student-tests/test_api.py::test_malformed_json_is_400[[]] PASSED [ 6%]
student-tests/test_api.py::test_malformed_json_is_400[null] PASSED [ 6%]
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}] PASSED [ 7%]
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}] PASSED [ 7%]
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "\ud800", "password": "Twelve-chars"}] PASSED [ 7%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body0] PASSED [ 8%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body1] PASSED [ 8%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body2] PASSED [ 8%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body3] PASSED [ 9%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body4] PASSED [ 9%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body5] PASSED [ 9%]
student-tests/test_api.py::test_unicode_limits_are_code_points_and_nul_is_stored PASSED [ 10%]
student-tests/test_api.py::test_oversized_body_and_wrong_content_type PASSED [ 10%]
student-tests/test_api.py::test_chat_validation PASSED [ 10%]
student-tests/test_api.py::test_foreign_chat_is_indistinguishable_from_missing PASSED [ 10%]
student-tests/test_api.py::test_chat_list_order_pagination_and_isolation PASSED [ 11%]
student-tests/test_api.py::test_history_pages_without_loss_or_duplicates PASSED [ 11%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=0] PASSED [ 11%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=101] PASSED [ 12%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=abc] PASSED [ 12%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=1&limit=2] PASSED [ 12%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=] PASSED [ 13%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=garbage] PASSED [ 13%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDF8Mg] PASSED [ 13%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDk5OTk5OTk5OTk5OTk5OTk5OTk5fHk=] PASSED [ 14%]
student-tests/test_api.py::test_cursor_bound_to_chat PASSED [ 14%]
student-tests/test_api.py::test_read_after_write PASSED [ 14%]
student-tests/test_api.py::test_malformed_cursor_values_are_400_on_both_lists PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[\xff] PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"first","TEXT":"second"}] PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"a","text":"b"}] PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":123}] PASSED [ 16%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}}] PASSED [ 16%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}]] PASSED [ 16%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x"} {}] PASSED [ 17%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}] PASSED [ 17%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":null}] PASSED [ 17%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":["nope"]}] PASSED [ 18%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":" \n\t "}] PASSED [ 18%]
student-tests/test_api.py::test_ff_byte_body_on_registration_is_contract_error PASSED [ 18%]
student-tests/test_api.py::test_text_is_stored_verbatim_and_length_counts_raw_text PASSED [ 19%]
student-tests/test_api.py::test_unknown_attachment_ids_are_404 PASSED [ 19%]
student-tests/test_api.py::test_member_ids_are_compared_as_uuids_not_strings PASSED [ 19%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer .] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer a b] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[bearer\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 21%]
student-tests/test_api.py::test_login_with_a_body_is_rejected PASSED [ 21%]
student-tests/test_api.py::test_password_range_is_12_to_128_code_points PASSED [ 21%]
student-tests/test_api.py::test_every_api_response_carries_instance_and_request_id PASSED [ 22%]
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\xa0] PASSED [ 22%]
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\u2003\u2003] PASSED [ 22%]
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[ \xa0\t\u3000] PASSED [ 23%]
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":"x","password":1e1000}] PASSED [ 23%]
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"ff_probe","display_name":"x","password":"12345678901\xff"}] PASSED [ 23%]
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}] PASSED [ 24%]
student-tests/test_api.py::test_more_than_100_members_is_400 PASSED [ 24%]
student-tests/test_api.py::test_cursor_of_one_list_is_rejected_by_the_other PASSED [ 24%]
student-tests/test_attachments.py::test_png_and_jpeg_roundtrip_byte_for_byte PASSED [ 25%]
student-tests/test_attachments.py::test_metadata_reveals_no_storage_details PASSED [ 25%]
student-tests/test_attachments.py::test_original_is_not_stored_on_api_containers PASSED [ 25%]
student-tests/test_attachments.py::test_outsider_and_unknown_get_404 PASSED [ 25%]
student-tests/test_attachments.py::test_attachment_ids_are_checked_per_chat PASSED [ 26%]
student-tests/test_attachments.py::test_message_with_attachments_and_history PASSED [ 26%]
student-tests/test_attachments.py::test_idempotent_replay_with_attachments PASSED [ 26%]
student-tests/test_attachments.py::test_message_body_validation[{"attachment_ids":["%(a)s"]}] PASSED [ 27%]
student-tests/test_attachments.py::test_message_body_validation[{"text":null,"attachment_ids":["%(a)s"]}] PASSED [ 27%]
student-tests/test_attachments.py::test_message_body_validation[{"text":1,"attachment_ids":["%(a)s"]}] PASSED [ 27%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":null}] PASSED [ 28%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":"%(a)s"}] PASSED [ 28%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":{}}] PASSED [ 28%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s","%] PASSED [ 29%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s",1]] PASSED [ 29%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["not-a-uui] PASSED [ 29%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"","attachment_ids":[]}] PASSED [ 30%]
student-tests/test_attachments.py::test_message_body_validation[{"text":" "}] PASSED [ 30%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":[],"sender_] PASSED [ 30%]
student-tests/test_attachments.py::test_at_most_ten_attachments PASSED [ 30%]
student-tests/test_attachments.py::test_size_limit_through_the_load_balancer PASSED [ 31%]
student-tests/test_attachments.py::test_pixel_limit_before_decoding PASSED [ 31%]
student-tests/test_attachments.py::test_exactly_40_megapixels_is_accepted PASSED [ 31%]
student-tests/test_attachments.py::test_bad_content_is_415[png: first 33 bytes of the fixture] PASSED [ 32%]
student-tests/test_attachments.py::test_bad_content_is_415[png: truncated] PASSED [ 32%]
student-tests/test_attachments.py::test_bad_content_is_415[png: corrupt IDAT, valid CRC] PASSED [ 32%]
student-tests/test_attachments.py::test_bad_content_is_415[png: fewer rows than IHDR] PASSED [ 33%]
student-tests/test_attachments.py::test_bad_content_is_415[png: flipped byte] PASSED [ 33%]
student-tests/test_attachments.py::test_bad_content_is_415[png: bytes after IEND] PASSED [ 33%]
student-tests/test_attachments.py::test_bad_content_is_415[jpeg: no scan data after SOS] PASSED [ 34%]
student-tests/test_attachments.py::test_bad_content_is_415[jpeg: truncated] PASSED [ 34%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: png bytes declared jpeg] PASSED [ 34%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: jpeg bytes declared png] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: gif declared png] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: html declared png] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[type: text/plain] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[type: octet-stream] PASSED [ 36%]
student-tests/test_attachments.py::test_bad_content_is_415[type: multipart] PASSED [ 36%]
student-tests/test_attachments.py::test_bad_content_is_415[type: missing] PASSED [ 36%]
student-tests/test_attachments.py::test_empty_body_is_400 PASSED [ 37%]
student-tests/test_attachments.py::test_bad_filename_is_400[missing] PASSED [ 37%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=] PASSED [ 37%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=%20%20%20] PASSED [ 38%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=.] PASSED [ 38%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=..] PASSED [ 38%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%2Fb.png] PASSED [ 39%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%5Cb.png] PASSED [ 39%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%00b.png] PASSED [ 39%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=c1%C2%85.png] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=tab%09.png] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=photo%E2%80%AEgnp.exe] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=%FF%FE.png] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a.png&filename=b.png] PASSED [ 41%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 41%]
student-tests/test_attachments.py::test_unicode_filename_is_kept_and_safely_exposed PASSED [ 41%]
student-tests/test_attachments.py::test_variant_parameter PASSED [ 42%]
student-tests/test_balancing.py::test_one_session_is_served_by_every_real_replica PASSED [ 42%]
student-tests/test_balancing.py::test_write_on_one_replica_is_read_on_the_other PASSED [ 42%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-pg_hba] PASSED [ 43%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-network] PASSED [ 43%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-pg_hba] PASSED [ 43%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-network] PASSED [ 44%]
student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[0] PASSED [ 44%]
student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[1] PASSED [ 44%]
student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[0] PASSED [ 45%]
student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[1] PASSED [ 45%]
student-tests/test_balancing.py::test_graceful_shutdown_order_in_logs PASSED [ 45%]
student-tests/test_balancing.py::test_full_restart_lb_rediscovers_all_replicas PASSED [ 45%]
student-tests/test_balancing.py::test_scale_up_and_down_is_picked_up_without_errors PASSED [ 46%]
student-tests/test_balancing.py::test_compose_policy_limits_ports_secrets_images PASSED [ 46%]
student-tests/test_balancing.py::test_running_containers_have_limits_and_no_bypass_ports PASSED [ 46%]
student-tests/test_balancing.py::test_migrations_ran_once_by_the_migrate_service PASSED [ 47%]
student-tests/test_consistency.py::test_interrupted_and_stalled_uploads_leave_nothing PASSED [ 47%]
student-tests/test_consistency.py::test_crash_between_s3_put_and_commit_is_cleaned_by_janitor PASSED [ 47%]
student-tests/test_consistency.py::test_lost_answer_after_commit_never_deletes_the_object PASSED [ 48%]
student-tests/test_consistency.py::test_fresh_pending_upload_is_not_touched_by_janitor PASSED [ 48%]
student-tests/test_db.py::test_registration_race_exactly_one_201_and_seven_409 PASSED [ 48%]
student-tests/test_db.py::test_create_chat_rolls_back_after_the_chat_row_was_inserted PASSED [ 49%]
student-tests/test_db.py::test_migrations_rerun_is_a_noop PASSED [ 49%]
student-tests/test_db.py::test_100_messages_from_10_clients_keep_order_and_cursor_walk PASSED [ 49%]
student-tests/test_db.py::test_secrets_are_not_stored_in_plain_text PASSED [ 50%]
student-tests/test_explain.py::test_late_page_uses_index_range_not_filter PASSED [ 50%]
student-tests/test_failures.py::test_valkey_failure[stop] PASSED [ 50%]
student-tests/test_failures.py::test_valkey_failure[pause] PASSED [ 50%]
student-tests/test_failures.py::test_postgres_failure[stop] PASSED [ 51%]
student-tests/test_failures.py::test_postgres_failure[pause] PASSED [ 51%]
student-tests/test_failures.py::test_locked_table_gives_bounded_503 PASSED [ 51%]
student-tests/test_failures.py::test_logout_during_valkey_outage_is_never_lost PASSED [ 52%]
student-tests/test_idempotency.py::test_replay_returns_the_same_201_and_message_from_every_replica PASSED [ 52%]
student-tests/test_idempotency.py::test_other_body_with_the_same_key_is_409_and_keeps_the_original PASSED [ 52%]
student-tests/test_idempotency.py::test_key_order_and_missing_attachment_ids_are_the_same_body PASSED [ 53%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[] PASSED [ 53%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 53%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[\u043a\u043b\u044e\u0447] PASSED [ 54%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a b] PASSED [ 54%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a/b] PASSED [ 54%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a,b] PASSED [ 55%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a\tb] PASSED [ 55%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[*] PASSED [ 55%]
student-tests/test_idempotency.py::test_repeated_key_header_is_400 PASSED [ 55%]
student-tests/test_idempotency.py::test_longest_key_and_every_allowed_character PASSED [ 56%]
student-tests/test_idempotency.py::test_scope_is_user_plus_chat_plus_key PASSED [ 56%]
student-tests/test_idempotency.py::test_outsider_and_revoked_session_never_see_the_stored_result PASSED [ 56%]
student-tests/test_idempotency.py::test_concurrent_same_key_through_different_replicas_creates_exactly_one PASSED [ 57%]
student-tests/test_idempotency.py::test_concurrent_same_key_different_bodies_one_wins PASSED [ 57%]
student-tests/test_idempotency.py::test_failure_after_the_key_insert_rolls_everything_back PASSED [ 57%]
student-tests/test_idempotency.py::test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message PASSED [ 58%]
student-tests/test_idempotency.py::test_lock_timeout_is_a_transient_503_that_is_not_recorded PASSED [ 58%]
student-tests/test_idempotency.py::test_replay_after_the_serving_replica_is_stopped PASSED [ 58%]
student-tests/test_reconcile.py::test_detect_and_safely_fix_faults_on_both_sides PASSED [ 59%]
student-tests/test_reconcile.py::test_checksum_mode_finds_same_size_corruption PASSED [ 59%]
student-tests/test_reconcile.py::test_safety_valve_refuses_mass_deletion PASSED [ 59%]
student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[s3] PASSED [ 60%]
student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[postgres] PASSED [ 60%]
student-tests/test_reconcile.py::test_live_uploads_are_never_hurt_by_concurrent_apply PASSED [ 60%]
student-tests/test_sessions.py::test_short_ttl_expires[valkey] PASSED [ 60%]
student-tests/test_sessions.py::test_short_ttl_expires[postgres] PASSED [ 61%]
student-tests/test_sessions.py::test_restart_does_not_extend_ttl FAILED [ 61%]
student-tests/test_sessions.py::test_postgres_session_store PASSED [ 61%]
student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop] FAILED [ 62%]
student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[pause] PASSED [ 62%]
student-tests/test_storage_failures.py::test_upload_survives_failover_of_the_serving_replica PASSED [ 62%]
student-tests/test_storage_failures.py::test_bucket_is_private_outside_the_api PASSED [ 63%]
student-tests/test_storage_failures.py::test_s3_restart_keeps_objects PASSED [ 63%]
student-tests/unit/test_images.py::test_public_fixture_is_accepted PASSED [ 63%]
student-tests/unit/test_images.py::test_valid_png_kinds[rgba] PASSED [ 64%]
student-tests/unit/test_images.py::test_valid_png_kinds[palette] PASSED [ 64%]
student-tests/unit/test_images.py::test_valid_png_kinds[1bit] PASSED [ 64%]
student-tests/unit/test_images.py::test_valid_png_kinds[gray] PASSED [ 65%]
student-tests/unit/test_images.py::test_valid_png_kinds[noise] PASSED [ 65%]
student-tests/unit/test_images.py::test_valid_png_kinds[1x1] PASSED [ 65%]
student-tests/unit/test_images.py::test_interlaced_png_is_accepted PASSED [ 65%]
student-tests/unit/test_images.py::test_broken_png_is_415[first 33 bytes of the fixture] PASSED [ 66%]
student-tests/unit/test_images.py::test_broken_png_is_415[truncated (no IEND)] PASSED [ 66%]
student-tests/unit/test_images.py::test_broken_png_is_415[truncated in IDAT] PASSED [ 66%]
student-tests/unit/test_images.py::test_broken_png_is_415[flipped byte -> CRC error] PASSED [ 67%]
student-tests/unit/test_images.py::test_broken_png_is_415[garbage IDAT with valid CRC] PASSED [ 67%]
student-tests/unit/test_images.py::test_broken_png_is_415[fewer rows than IHDR] PASSED [ 67%]
student-tests/unit/test_images.py::test_broken_png_is_415[bytes after IEND (polyglot)] PASSED [ 68%]
student-tests/unit/test_images.py::test_broken_png_is_415[text] PASSED [ 68%]
student-tests/unit/test_images.py::test_broken_png_is_415[gif] PASSED [ 68%]
student-tests/unit/test_images.py::test_broken_png_is_415[two bytes of a signature] PASSED [ 69%]
student-tests/unit/test_images.py::test_pixel_limit_is_checked_from_the_header PASSED [ 69%]
student-tests/unit/test_images.py::test_png_verification_is_streaming_and_fast PASSED [ 69%]
student-tests/unit/test_images.py::test_jpeg_header_is_parsed_incrementally PASSED [ 70%]
student-tests/unit/test_images.py::test_valid_jpeg[base] SKIPPED (dj...) [ 70%]
student-tests/unit/test_images.py::test_valid_jpeg[prog] SKIPPED (dj...) [ 70%]
student-tests/unit/test_images.py::test_broken_jpeg_is_415[no scan data after SOS-] SKIPPED [ 70%]
student-tests/unit/test_images.py::test_broken_jpeg_is_415[truncated-] SKIPPED [ 71%]
student-tests/unit/test_images.py::test_good_filenames[photo.png] PASSED [ 71%]
student-tests/unit/test_images.py::test_good_filenames[\u0444\u043e\u0442\u043e \u043e\u0442\u043f\u0443\u0441\u043a\u0430.jpg] PASSED [ 71%]
student-tests/unit/test_images.py::test_good_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 72%]
student-tests/unit/test_images.py::test_good_filenames[emoji \U0001f468\u200d\U0001f469\u200d\U0001f467.png] PASSED [ 72%]
student-tests/unit/test_images.py::test_good_filenames[ x ] PASSED [ 72%]
student-tests/unit/test_images.py::test_good_filenames[..x] PASSED [ 73%]
student-tests/unit/test_images.py::test_bad_filenames[] PASSED [ 73%]
student-tests/unit/test_images.py::test_bad_filenames[ ] PASSED [ 73%]
student-tests/unit/test_images.py::test_bad_filenames[\u3000] PASSED [ 74%]
student-tests/unit/test_images.py::test_bad_filenames[.] PASSED [ 74%]
student-tests/unit/test_images.py::test_bad_filenames[..] PASSED [ 74%]
student-tests/unit/test_images.py::test_bad_filenames[a/b.png] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[a\\b.png] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[a\x00b] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[c1\x85.png] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[tab\t.png] PASSED [ 76%]
student-tests/unit/test_images.py::test_bad_filenames[del\x7f.png] PASSED [ 76%]
student-tests/unit/test_images.py::test_bad_filenames[x\u2028y] PASSED [ 76%]
student-tests/unit/test_images.py::test_bad_filenames[photo\u202egnp.exe] PASSED [ 77%]
student-tests/unit/test_images.py::test_bad_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 77%]
student-tests/unit/test_images.py::test_filename_must_be_single_and_present PASSED [ 77%]
student-tests/unit/test_images.py::test_query_must_be_valid_utf8 PASSED [ 78%]
student-tests/unit/test_unit.py::test_valid_passwords[Twelve-chars] PASSED [ 78%]
student-tests/unit/test_unit.py::test_valid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 78%]
student-tests/unit/test_unit.py::test_valid_passwords[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u044f\u043c\u0438] PASSED [ 79%]
student-tests/unit/test_unit.py::test_valid_passwords[\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600] PASSED [ 79%]
student-tests/unit/test_unit.py::test_valid_passwords[a b c d e f g] PASSED [ 79%]
student-tests/unit/test_unit.py::test_invalid_passwords[short-11ch] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[ ] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800] PASSED [ 81%]
student-tests/unit/test_unit.py::test_invalid_passwords[None] PASSED [ 81%]
student-tests/unit/test_unit.py::test_invalid_passwords[123456789012] PASSED [ 81%]
student-tests/unit/test_unit.py::test_text_may_contain_nul_but_not_lone_surrogates PASSED [ 82%]
student-tests/unit/test_unit.py::test_cursor_roundtrip_and_binding PASSED [ 82%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|253402300800000000|11111111-1111-4111-8111-111111111111] PASSED [ 82%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|9999999999999999999|11111111-1111-4111-8111-111111111111] PASSED [ 83%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|-1|11111111-1111-4111-8111-111111111111] PASSED [ 83%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|not-a-uuid] PASSED [ 83%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-11111111111G] PASSED [ 84%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v2|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-111111111111] PASSED [ 84%]
student-tests/unit/test_unit.py::test_time_helpers_are_exact PASSED [ 84%]
student-tests/unit/test_unit.py::test_basic_is_parsed_and_password_may_contain_colons PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[None] PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[] PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic] PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic \xe9] PASSED [ 86%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic !!!!] PASSED [ 86%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic bm8tY29sb24=] PASSED [ 86%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic //46cHc=] PASSED [ 87%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic OnBhc3N3b3Jk] PASSED [ 87%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic YWxpY2U6] PASSED [ 87%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic 0JDQu9C40YHQsDpwdw==] PASSED [ 88%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 88%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA] PASSED [ 88%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[None] PASSED [ 89%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[] PASSED [ 89%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer] PASSED [ 89%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer short] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Basic YWxpY2U6cGFzc3dvcmQtMTIz] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 91%]
student-tests/unit/test_unit.py::test_bearer_ok PASSED [ 91%]
student-tests/unit/test_unit.py::test_argon2id_hash_and_verify PASSED [ 91%]
student-tests/unit/test_unit.py::test_s3_settings_are_validated_and_contract_limits_cannot_be_raised PASSED [ 92%]
student-tests/unit/test_unit.py::test_settings_read_secret_files_and_contract_names PASSED [ 92%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[a] PASSED [ 92%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 93%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[A-Za-z0-9._:-] PASSED [ 93%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[2026-10-16T10:00:00.1] PASSED [ 93%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[None] PASSED [ 94%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[] PASSED [ 94%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 94%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\u043a\u043b\u044e\u0447] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a b] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a/b] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a,b] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\tb] PASSED [ 96%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\x00b] PASSED [ 96%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[*] PASSED [ 96%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\xe9] PASSED [ 97%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"a","text":"b"}] PASSED [ 97%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"a":{"x":1,"x":2}}] PASSED [ 97%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[\xff] PASSED [ 98%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"x"} {}] PASSED [ 98%]
student-tests/unit/test_unit.py::test_attachment_ids_absent_equals_empty_and_is_validated PASSED [ 98%]
student-tests/unit/test_unit.py::test_request_fingerprint_ignores_key_order_and_default_attachments PASSED [ 99%]
student-tests/unit/test_unit.py::test_instance_id_defaults_to_hostname_and_is_validated PASSED [ 99%]
student-tests/unit/test_unit.py::test_idempotency_ttl_cannot_be_below_24_hours PASSED [ 99%]
student-tests/unit/test_unit.py::test_message_write_path_setting PASSED [100%]
================================== FAILURES ===================================
______________________ test_restart_does_not_extend_ttl _______________________
api = <conftest.Api object at 0x000002244DC7BCB0>
def test_restart_does_not_extend_ttl(api):
with api_with_env(SESSION_TTL_SECONDS="12"):
actor = api.register("ttlrestart")
session = login_session(api, actor)
restart_services("valkey", "api")
> assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it"
E AssertionError: restart took longer than the TTL; increase it
E assert -0.024317 > 0
E + where -0.024317 = seconds_left('2026-10-25T12:00:02.431630Z')
student-tests\test_sessions.py:45: AssertionError
_____________ test_s3_outage_is_503_and_text_keeps_working[stop] ______________
api = <conftest.Api object at 0x000002244DC7BCB0>
pair = ({'id': '4b97c381-df7f-48f6-92df-bb7a17232630', 'user': {'id': '4b97c381-df7f-48f6-92df-bb7a17232630', 'username': 'al...81-df7f-48f6-92df-bb7a17232630', '1a1510cc-7f0b-423f-8645-70e2fa838cb2'], 'created_at': '2026-10-25T12:01:26.054625Z'})
how = 'stop'
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
"""S3 down (stopped: connection refused; paused: no answer at all):
* readiness is 503 on every replica (and through the LB), quickly;
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
* after recovery the earlier file is intact."""
alice, bob, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
with service_down("s3", how):
deadline = time.monotonic() + 15
while api.client.get("/health/ready").status_code != 503:
assert time.monotonic() < deadline, "readiness did not turn 503"
time.sleep(0.5)
for replica in running_replicas():
assert probe_inside(replica, "/health/ready") == 503
assert probe_inside(replica, "/health/ready/core") == 200
started = time.monotonic()
error = api.upload(alice, chat, SAMPLE, 503).json()
assert error["error"]["code"] == "unavailable"
assert time.monotonic() - started < 8
started = time.monotonic()
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
assert time.monotonic() - started < 8
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
api.send(alice, chat, "текст работает без S3") # degraded mode
> got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
student-tests\test_storage_failures.py:53:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <conftest.Api object at 0x000002244DC7BCB0>, method = 'GET'
path = '/api/v1/attachments/57b6e686-85b7-4dd3-8e02-076b19a8ca79/content'
expected = 200, token = 'S12UUVFiNSnc1NKWb-5PmdJBo9_J6P1-5TihoVd7xBs'
json_body = None, raw = None, headers = None, schema = None, lb_error_ok = False
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
lb_error_ok: bool = False):
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
accepted without X-Instance-Id; every response of an API replica must carry it."""
h = dict(headers or {})
if token is not None:
h["Authorization"] = "Bearer " + token
if json_body is not None:
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
response = self.client.request(method, path, content=raw, headers=h)
allowed = (expected,) if isinstance(expected, int) else expected
> assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
E AssertionError: ('GET', '/api/v1/attachments/57b6e686-85b7-4dd3-8e02-076b19a8ca79/content', 503, '{"error":{"code":"unavailable","message":"Хранилище файлов временно недоступно","request_id":"8ecf9351-b1db-4e6d-99da-89f4de083683"}}')
E assert 503 in (200,)
E + where 503 = <Response [503 Service Unavailable]>.status_code
student-tests\conftest.py:184: AssertionError
=========================== short test summary info ===========================
FAILED student-tests/test_sessions.py::test_restart_does_not_extend_ttl - Ass...
FAILED student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop]
============ 2 failed, 314 passed, 4 skipped in 1229.88s (0:20:29) ============
[exit 1, 1231.4 s]
+331
View File
@@ -0,0 +1,331 @@
# Own tests after fixes found by the first clean-clone run
# 2026-10-25T12:12:45Z, commit fc50362 bench: SQL синтетики через stdin psql (лимит командной строки Windows)
$ python -m pytest student-tests -v
============================= test session starts =============================
platform win32 -- Python 3.14.6, pytest-9.1.1, pluggy-1.7.0 -- C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe
rootdir: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean
plugins: anyio-4.15.1
collecting ... collected 320 items
student-tests/test_api.py::test_register_never_returns_password_and_login_returns_bearer PASSED [ 0%]
student-tests/test_api.py::test_duplicate_username_is_409 PASSED [ 0%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[short-11ch] PASSED [ 0%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[ ] PASSED [ 1%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[ \t \n \t \n \t \n ] PASSED [ 1%]
student-tests/test_api.py::test_bad_password_on_registration_is_400[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 1%]
student-tests/test_api.py::test_edge_passwords_register_and_login[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 2%]
student-tests/test_api.py::test_edge_passwords_register_and_login[with:colons:inside] PASSED [ 2%]
student-tests/test_api.py::test_edge_passwords_register_and_login[Twelve chars] PASSED [ 2%]
student-tests/test_api.py::test_wrong_password_and_unknown_user_give_the_same_401 PASSED [ 3%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic \xe9] PASSED [ 3%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic !!!] PASSED [ 3%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic bm9fY29sb24=] PASSED [ 4%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic //46eA==] PASSED [ 4%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 4%]
student-tests/test_api.py::test_malformed_basic_is_401_not_500[] PASSED [ 5%]
student-tests/test_api.py::test_logout_revokes_only_this_token PASSED [ 5%]
student-tests/test_api.py::test_x_user_id_basic_and_garbage_do_not_authenticate PASSED [ 5%]
student-tests/test_api.py::test_malformed_json_is_400[{] PASSED [ 5%]
student-tests/test_api.py::test_malformed_json_is_400[] PASSED [ 6%]
student-tests/test_api.py::test_malformed_json_is_400[[]] PASSED [ 6%]
student-tests/test_api.py::test_malformed_json_is_400[null] PASSED [ 6%]
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}] PASSED [ 7%]
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}] PASSED [ 7%]
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "\ud800", "password": "Twelve-chars"}] PASSED [ 7%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body0] PASSED [ 8%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body1] PASSED [ 8%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body2] PASSED [ 8%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body3] PASSED [ 9%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body4] PASSED [ 9%]
student-tests/test_api.py::test_invalid_registration_fields_are_400[body5] PASSED [ 9%]
student-tests/test_api.py::test_unicode_limits_are_code_points_and_nul_is_stored PASSED [ 10%]
student-tests/test_api.py::test_oversized_body_and_wrong_content_type PASSED [ 10%]
student-tests/test_api.py::test_chat_validation PASSED [ 10%]
student-tests/test_api.py::test_foreign_chat_is_indistinguishable_from_missing PASSED [ 10%]
student-tests/test_api.py::test_chat_list_order_pagination_and_isolation PASSED [ 11%]
student-tests/test_api.py::test_history_pages_without_loss_or_duplicates PASSED [ 11%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=0] PASSED [ 11%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=101] PASSED [ 12%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=abc] PASSED [ 12%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=1&limit=2] PASSED [ 12%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=] PASSED [ 13%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=garbage] PASSED [ 13%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDF8Mg] PASSED [ 13%]
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDk5OTk5OTk5OTk5OTk5OTk5OTk5fHk=] PASSED [ 14%]
student-tests/test_api.py::test_cursor_bound_to_chat PASSED [ 14%]
student-tests/test_api.py::test_read_after_write PASSED [ 14%]
student-tests/test_api.py::test_malformed_cursor_values_are_400_on_both_lists PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[\xff] PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"first","TEXT":"second"}] PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"a","text":"b"}] PASSED [ 15%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":123}] PASSED [ 16%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}}] PASSED [ 16%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}]] PASSED [ 16%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x"} {}] PASSED [ 17%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}] PASSED [ 17%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":null}] PASSED [ 17%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":["nope"]}] PASSED [ 18%]
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":" \n\t "}] PASSED [ 18%]
student-tests/test_api.py::test_ff_byte_body_on_registration_is_contract_error PASSED [ 18%]
student-tests/test_api.py::test_text_is_stored_verbatim_and_length_counts_raw_text PASSED [ 19%]
student-tests/test_api.py::test_unknown_attachment_ids_are_404 PASSED [ 19%]
student-tests/test_api.py::test_member_ids_are_compared_as_uuids_not_strings PASSED [ 19%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer .] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer a b] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 20%]
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[bearer\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 21%]
student-tests/test_api.py::test_login_with_a_body_is_rejected PASSED [ 21%]
student-tests/test_api.py::test_password_range_is_12_to_128_code_points PASSED [ 21%]
student-tests/test_api.py::test_every_api_response_carries_instance_and_request_id PASSED [ 22%]
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\xa0] PASSED [ 22%]
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\u2003\u2003] PASSED [ 22%]
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[ \xa0\t\u3000] PASSED [ 23%]
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":"x","password":1e1000}] PASSED [ 23%]
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"ff_probe","display_name":"x","password":"12345678901\xff"}] PASSED [ 23%]
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}] PASSED [ 24%]
student-tests/test_api.py::test_more_than_100_members_is_400 PASSED [ 24%]
student-tests/test_api.py::test_cursor_of_one_list_is_rejected_by_the_other PASSED [ 24%]
student-tests/test_attachments.py::test_png_and_jpeg_roundtrip_byte_for_byte PASSED [ 25%]
student-tests/test_attachments.py::test_metadata_reveals_no_storage_details PASSED [ 25%]
student-tests/test_attachments.py::test_original_is_not_stored_on_api_containers PASSED [ 25%]
student-tests/test_attachments.py::test_outsider_and_unknown_get_404 PASSED [ 25%]
student-tests/test_attachments.py::test_attachment_ids_are_checked_per_chat PASSED [ 26%]
student-tests/test_attachments.py::test_message_with_attachments_and_history PASSED [ 26%]
student-tests/test_attachments.py::test_idempotent_replay_with_attachments PASSED [ 26%]
student-tests/test_attachments.py::test_message_body_validation[{"attachment_ids":["%(a)s"]}] PASSED [ 27%]
student-tests/test_attachments.py::test_message_body_validation[{"text":null,"attachment_ids":["%(a)s"]}] PASSED [ 27%]
student-tests/test_attachments.py::test_message_body_validation[{"text":1,"attachment_ids":["%(a)s"]}] PASSED [ 27%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":null}] PASSED [ 28%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":"%(a)s"}] PASSED [ 28%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":{}}] PASSED [ 28%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s","%] PASSED [ 29%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s",1]] PASSED [ 29%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["not-a-uui] PASSED [ 29%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"","attachment_ids":[]}] PASSED [ 30%]
student-tests/test_attachments.py::test_message_body_validation[{"text":" "}] PASSED [ 30%]
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":[],"sender_] PASSED [ 30%]
student-tests/test_attachments.py::test_at_most_ten_attachments PASSED [ 30%]
student-tests/test_attachments.py::test_size_limit_through_the_load_balancer PASSED [ 31%]
student-tests/test_attachments.py::test_pixel_limit_before_decoding PASSED [ 31%]
student-tests/test_attachments.py::test_exactly_40_megapixels_is_accepted PASSED [ 31%]
student-tests/test_attachments.py::test_bad_content_is_415[png: first 33 bytes of the fixture] PASSED [ 32%]
student-tests/test_attachments.py::test_bad_content_is_415[png: truncated] PASSED [ 32%]
student-tests/test_attachments.py::test_bad_content_is_415[png: corrupt IDAT, valid CRC] PASSED [ 32%]
student-tests/test_attachments.py::test_bad_content_is_415[png: fewer rows than IHDR] PASSED [ 33%]
student-tests/test_attachments.py::test_bad_content_is_415[png: flipped byte] PASSED [ 33%]
student-tests/test_attachments.py::test_bad_content_is_415[png: bytes after IEND] PASSED [ 33%]
student-tests/test_attachments.py::test_bad_content_is_415[jpeg: no scan data after SOS] PASSED [ 34%]
student-tests/test_attachments.py::test_bad_content_is_415[jpeg: truncated] PASSED [ 34%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: png bytes declared jpeg] PASSED [ 34%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: jpeg bytes declared png] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: gif declared png] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[mime: html declared png] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[type: text/plain] PASSED [ 35%]
student-tests/test_attachments.py::test_bad_content_is_415[type: octet-stream] PASSED [ 36%]
student-tests/test_attachments.py::test_bad_content_is_415[type: multipart] PASSED [ 36%]
student-tests/test_attachments.py::test_bad_content_is_415[type: missing] PASSED [ 36%]
student-tests/test_attachments.py::test_empty_body_is_400 PASSED [ 37%]
student-tests/test_attachments.py::test_bad_filename_is_400[missing] PASSED [ 37%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=] PASSED [ 37%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=%20%20%20] PASSED [ 38%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=.] PASSED [ 38%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=..] PASSED [ 38%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%2Fb.png] PASSED [ 39%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%5Cb.png] PASSED [ 39%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%00b.png] PASSED [ 39%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=c1%C2%85.png] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=tab%09.png] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=photo%E2%80%AEgnp.exe] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=%FF%FE.png] PASSED [ 40%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a.png&filename=b.png] PASSED [ 41%]
student-tests/test_attachments.py::test_bad_filename_is_400[filename=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 41%]
student-tests/test_attachments.py::test_unicode_filename_is_kept_and_safely_exposed PASSED [ 41%]
student-tests/test_attachments.py::test_variant_parameter PASSED [ 42%]
student-tests/test_balancing.py::test_one_session_is_served_by_every_real_replica PASSED [ 42%]
student-tests/test_balancing.py::test_write_on_one_replica_is_read_on_the_other PASSED [ 42%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-pg_hba] PASSED [ 43%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-network] PASSED [ 43%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-pg_hba] PASSED [ 43%]
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-network] PASSED [ 44%]
student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[0] PASSED [ 44%]
student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[1] PASSED [ 44%]
student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[0] PASSED [ 45%]
student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[1] PASSED [ 45%]
student-tests/test_balancing.py::test_graceful_shutdown_order_in_logs PASSED [ 45%]
student-tests/test_balancing.py::test_full_restart_lb_rediscovers_all_replicas PASSED [ 45%]
student-tests/test_balancing.py::test_scale_up_and_down_is_picked_up_without_errors PASSED [ 46%]
student-tests/test_balancing.py::test_compose_policy_limits_ports_secrets_images PASSED [ 46%]
student-tests/test_balancing.py::test_running_containers_have_limits_and_no_bypass_ports PASSED [ 46%]
student-tests/test_balancing.py::test_migrations_ran_once_by_the_migrate_service PASSED [ 47%]
student-tests/test_consistency.py::test_interrupted_and_stalled_uploads_leave_nothing PASSED [ 47%]
student-tests/test_consistency.py::test_crash_between_s3_put_and_commit_is_cleaned_by_janitor PASSED [ 47%]
student-tests/test_consistency.py::test_lost_answer_after_commit_never_deletes_the_object PASSED [ 48%]
student-tests/test_consistency.py::test_fresh_pending_upload_is_not_touched_by_janitor PASSED [ 48%]
student-tests/test_db.py::test_registration_race_exactly_one_201_and_seven_409 PASSED [ 48%]
student-tests/test_db.py::test_create_chat_rolls_back_after_the_chat_row_was_inserted PASSED [ 49%]
student-tests/test_db.py::test_migrations_rerun_is_a_noop PASSED [ 49%]
student-tests/test_db.py::test_100_messages_from_10_clients_keep_order_and_cursor_walk PASSED [ 49%]
student-tests/test_db.py::test_secrets_are_not_stored_in_plain_text PASSED [ 50%]
student-tests/test_explain.py::test_late_page_uses_index_range_not_filter PASSED [ 50%]
student-tests/test_failures.py::test_valkey_failure[stop] PASSED [ 50%]
student-tests/test_failures.py::test_valkey_failure[pause] PASSED [ 50%]
student-tests/test_failures.py::test_postgres_failure[stop] PASSED [ 51%]
student-tests/test_failures.py::test_postgres_failure[pause] PASSED [ 51%]
student-tests/test_failures.py::test_locked_table_gives_bounded_503 PASSED [ 51%]
student-tests/test_failures.py::test_logout_during_valkey_outage_is_never_lost PASSED [ 52%]
student-tests/test_idempotency.py::test_replay_returns_the_same_201_and_message_from_every_replica PASSED [ 52%]
student-tests/test_idempotency.py::test_other_body_with_the_same_key_is_409_and_keeps_the_original PASSED [ 52%]
student-tests/test_idempotency.py::test_key_order_and_missing_attachment_ids_are_the_same_body PASSED [ 53%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[] PASSED [ 53%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 53%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[\u043a\u043b\u044e\u0447] PASSED [ 54%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a b] PASSED [ 54%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a/b] PASSED [ 54%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a,b] PASSED [ 55%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a\tb] PASSED [ 55%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[*] PASSED [ 55%]
student-tests/test_idempotency.py::test_repeated_key_header_is_400 PASSED [ 55%]
student-tests/test_idempotency.py::test_longest_key_and_every_allowed_character PASSED [ 56%]
student-tests/test_idempotency.py::test_scope_is_user_plus_chat_plus_key PASSED [ 56%]
student-tests/test_idempotency.py::test_outsider_and_revoked_session_never_see_the_stored_result PASSED [ 56%]
student-tests/test_idempotency.py::test_concurrent_same_key_through_different_replicas_creates_exactly_one PASSED [ 57%]
student-tests/test_idempotency.py::test_concurrent_same_key_different_bodies_one_wins PASSED [ 57%]
student-tests/test_idempotency.py::test_failure_after_the_key_insert_rolls_everything_back PASSED [ 57%]
student-tests/test_idempotency.py::test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message PASSED [ 58%]
student-tests/test_idempotency.py::test_lock_timeout_is_a_transient_503_that_is_not_recorded PASSED [ 58%]
student-tests/test_idempotency.py::test_replay_after_the_serving_replica_is_stopped PASSED [ 58%]
student-tests/test_reconcile.py::test_detect_and_safely_fix_faults_on_both_sides PASSED [ 59%]
student-tests/test_reconcile.py::test_checksum_mode_finds_same_size_corruption PASSED [ 59%]
student-tests/test_reconcile.py::test_safety_valve_refuses_mass_deletion PASSED [ 59%]
student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[s3] PASSED [ 60%]
student-tests/test_reconcile.py::test_store_outage_stops_the_run_without_damage[postgres] PASSED [ 60%]
student-tests/test_reconcile.py::test_live_uploads_are_never_hurt_by_concurrent_apply PASSED [ 60%]
student-tests/test_sessions.py::test_short_ttl_expires[valkey] PASSED [ 60%]
student-tests/test_sessions.py::test_short_ttl_expires[postgres] PASSED [ 61%]
student-tests/test_sessions.py::test_restart_does_not_extend_ttl PASSED [ 61%]
student-tests/test_sessions.py::test_postgres_session_store PASSED [ 61%]
student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop] PASSED [ 62%]
student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[pause] PASSED [ 62%]
student-tests/test_storage_failures.py::test_upload_survives_failover_of_the_serving_replica PASSED [ 62%]
student-tests/test_storage_failures.py::test_bucket_is_private_outside_the_api PASSED [ 63%]
student-tests/test_storage_failures.py::test_s3_restart_keeps_objects PASSED [ 63%]
student-tests/unit/test_images.py::test_public_fixture_is_accepted PASSED [ 63%]
student-tests/unit/test_images.py::test_valid_png_kinds[rgba] PASSED [ 64%]
student-tests/unit/test_images.py::test_valid_png_kinds[palette] PASSED [ 64%]
student-tests/unit/test_images.py::test_valid_png_kinds[1bit] PASSED [ 64%]
student-tests/unit/test_images.py::test_valid_png_kinds[gray] PASSED [ 65%]
student-tests/unit/test_images.py::test_valid_png_kinds[noise] PASSED [ 65%]
student-tests/unit/test_images.py::test_valid_png_kinds[1x1] PASSED [ 65%]
student-tests/unit/test_images.py::test_interlaced_png_is_accepted PASSED [ 65%]
student-tests/unit/test_images.py::test_broken_png_is_415[first 33 bytes of the fixture] PASSED [ 66%]
student-tests/unit/test_images.py::test_broken_png_is_415[truncated (no IEND)] PASSED [ 66%]
student-tests/unit/test_images.py::test_broken_png_is_415[truncated in IDAT] PASSED [ 66%]
student-tests/unit/test_images.py::test_broken_png_is_415[flipped byte -> CRC error] PASSED [ 67%]
student-tests/unit/test_images.py::test_broken_png_is_415[garbage IDAT with valid CRC] PASSED [ 67%]
student-tests/unit/test_images.py::test_broken_png_is_415[fewer rows than IHDR] PASSED [ 67%]
student-tests/unit/test_images.py::test_broken_png_is_415[bytes after IEND (polyglot)] PASSED [ 68%]
student-tests/unit/test_images.py::test_broken_png_is_415[text] PASSED [ 68%]
student-tests/unit/test_images.py::test_broken_png_is_415[gif] PASSED [ 68%]
student-tests/unit/test_images.py::test_broken_png_is_415[two bytes of a signature] PASSED [ 69%]
student-tests/unit/test_images.py::test_pixel_limit_is_checked_from_the_header PASSED [ 69%]
student-tests/unit/test_images.py::test_png_verification_is_streaming_and_fast PASSED [ 69%]
student-tests/unit/test_images.py::test_jpeg_header_is_parsed_incrementally PASSED [ 70%]
student-tests/unit/test_images.py::test_valid_jpeg[base] SKIPPED (dj...) [ 70%]
student-tests/unit/test_images.py::test_valid_jpeg[prog] SKIPPED (dj...) [ 70%]
student-tests/unit/test_images.py::test_broken_jpeg_is_415[no scan data after SOS-] SKIPPED [ 70%]
student-tests/unit/test_images.py::test_broken_jpeg_is_415[truncated-] SKIPPED [ 71%]
student-tests/unit/test_images.py::test_good_filenames[photo.png] PASSED [ 71%]
student-tests/unit/test_images.py::test_good_filenames[\u0444\u043e\u0442\u043e \u043e\u0442\u043f\u0443\u0441\u043a\u0430.jpg] PASSED [ 71%]
student-tests/unit/test_images.py::test_good_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 72%]
student-tests/unit/test_images.py::test_good_filenames[emoji \U0001f468\u200d\U0001f469\u200d\U0001f467.png] PASSED [ 72%]
student-tests/unit/test_images.py::test_good_filenames[ x ] PASSED [ 72%]
student-tests/unit/test_images.py::test_good_filenames[..x] PASSED [ 73%]
student-tests/unit/test_images.py::test_bad_filenames[] PASSED [ 73%]
student-tests/unit/test_images.py::test_bad_filenames[ ] PASSED [ 73%]
student-tests/unit/test_images.py::test_bad_filenames[\u3000] PASSED [ 74%]
student-tests/unit/test_images.py::test_bad_filenames[.] PASSED [ 74%]
student-tests/unit/test_images.py::test_bad_filenames[..] PASSED [ 74%]
student-tests/unit/test_images.py::test_bad_filenames[a/b.png] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[a\\b.png] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[a\x00b] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[c1\x85.png] PASSED [ 75%]
student-tests/unit/test_images.py::test_bad_filenames[tab\t.png] PASSED [ 76%]
student-tests/unit/test_images.py::test_bad_filenames[del\x7f.png] PASSED [ 76%]
student-tests/unit/test_images.py::test_bad_filenames[x\u2028y] PASSED [ 76%]
student-tests/unit/test_images.py::test_bad_filenames[photo\u202egnp.exe] PASSED [ 77%]
student-tests/unit/test_images.py::test_bad_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 77%]
student-tests/unit/test_images.py::test_filename_must_be_single_and_present PASSED [ 77%]
student-tests/unit/test_images.py::test_query_must_be_valid_utf8 PASSED [ 78%]
student-tests/unit/test_unit.py::test_valid_passwords[Twelve-chars] PASSED [ 78%]
student-tests/unit/test_unit.py::test_valid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 78%]
student-tests/unit/test_unit.py::test_valid_passwords[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u044f\u043c\u0438] PASSED [ 79%]
student-tests/unit/test_unit.py::test_valid_passwords[\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600] PASSED [ 79%]
student-tests/unit/test_unit.py::test_valid_passwords[a b c d e f g] PASSED [ 79%]
student-tests/unit/test_unit.py::test_invalid_passwords[short-11ch] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[ ] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000] PASSED [ 80%]
student-tests/unit/test_unit.py::test_invalid_passwords[\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800] PASSED [ 81%]
student-tests/unit/test_unit.py::test_invalid_passwords[None] PASSED [ 81%]
student-tests/unit/test_unit.py::test_invalid_passwords[123456789012] PASSED [ 81%]
student-tests/unit/test_unit.py::test_text_may_contain_nul_but_not_lone_surrogates PASSED [ 82%]
student-tests/unit/test_unit.py::test_cursor_roundtrip_and_binding PASSED [ 82%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|253402300800000000|11111111-1111-4111-8111-111111111111] PASSED [ 82%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|9999999999999999999|11111111-1111-4111-8111-111111111111] PASSED [ 83%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|-1|11111111-1111-4111-8111-111111111111] PASSED [ 83%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|not-a-uuid] PASSED [ 83%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-11111111111G] PASSED [ 84%]
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v2|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-111111111111] PASSED [ 84%]
student-tests/unit/test_unit.py::test_time_helpers_are_exact PASSED [ 84%]
student-tests/unit/test_unit.py::test_basic_is_parsed_and_password_may_contain_colons PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[None] PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[] PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic] PASSED [ 85%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic \xe9] PASSED [ 86%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic !!!!] PASSED [ 86%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic bm8tY29sb24=] PASSED [ 86%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic //46cHc=] PASSED [ 87%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic OnBhc3N3b3Jk] PASSED [ 87%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic YWxpY2U6] PASSED [ 87%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic 0JDQu9C40YHQsDpwdw==] PASSED [ 88%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 88%]
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA] PASSED [ 88%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[None] PASSED [ 89%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[] PASSED [ 89%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer] PASSED [ 89%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer short] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Basic YWxpY2U6cGFzc3dvcmQtMTIz] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 90%]
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 91%]
student-tests/unit/test_unit.py::test_bearer_ok PASSED [ 91%]
student-tests/unit/test_unit.py::test_argon2id_hash_and_verify PASSED [ 91%]
student-tests/unit/test_unit.py::test_s3_settings_are_validated_and_contract_limits_cannot_be_raised PASSED [ 92%]
student-tests/unit/test_unit.py::test_settings_read_secret_files_and_contract_names PASSED [ 92%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[a] PASSED [ 92%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 93%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[A-Za-z0-9._:-] PASSED [ 93%]
student-tests/unit/test_unit.py::test_valid_idempotency_keys[2026-10-16T10:00:00.1] PASSED [ 93%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[None] PASSED [ 94%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[] PASSED [ 94%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 94%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\u043a\u043b\u044e\u0447] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a b] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a/b] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a,b] PASSED [ 95%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\tb] PASSED [ 96%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\x00b] PASSED [ 96%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[*] PASSED [ 96%]
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\xe9] PASSED [ 97%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"a","text":"b"}] PASSED [ 97%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"a":{"x":1,"x":2}}] PASSED [ 97%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[\xff] PASSED [ 98%]
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"x"} {}] PASSED [ 98%]
student-tests/unit/test_unit.py::test_attachment_ids_absent_equals_empty_and_is_validated PASSED [ 98%]
student-tests/unit/test_unit.py::test_request_fingerprint_ignores_key_order_and_default_attachments PASSED [ 99%]
student-tests/unit/test_unit.py::test_instance_id_defaults_to_hostname_and_is_validated PASSED [ 99%]
student-tests/unit/test_unit.py::test_idempotency_ttl_cannot_be_below_24_hours PASSED [ 99%]
student-tests/unit/test_unit.py::test_message_write_path_setting PASSED [100%]
================= 316 passed, 4 skipped in 1269.76s (0:21:09) =================
+74
View File
@@ -0,0 +1,74 @@
# make test-failover with scripts/stop-one: which container is really stopped, restore, repeat
# 2026-10-25T11:42:11Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов
--- run 1; running API replicas BEFORE:
mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=33790 started=2026-10-25T11:41:59.187
mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=33848 started=2026-10-25T11:41:59.568
$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 5.956}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 16.5 s]
AFTER (the stopped replica, left stopped by the hook):
mtusi-lab4-api-1 host=8691c30aa702 exited exit=0 pid=0 started=2026-10-25T11:41:59.187
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api2 172.25.0.3 UP mtusi-lab4-api-2 X-Instance-Id=4026e296947c
(other slots: MAINT, no container)
[exit 0, 0.7 s]
$ sh scripts/restore-api.sh
mtusi-lab4-api-1 Up 6 seconds (healthy)
mtusi-lab4-api-2 Up 37 seconds (healthy)
Container mtusi-lab4-api-2 Running
Container mtusi-lab4-api-1 Starting
Container mtusi-lab4-api-1 Started
Container mtusi-lab4-api-1 Waiting
Container mtusi-lab4-api-2 Waiting
Container mtusi-lab4-api-1 Healthy
Container mtusi-lab4-api-2 Healthy
[exit 0, 6.9 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['4026e296947c', '8691c30aa702']
[exit 0, 1.4 s]
--- run 2; running API replicas BEFORE:
mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=35544 started=2026-10-25T11:42:30.519
mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=33848 started=2026-10-25T11:41:59.568
$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 6.291}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 16.4 s]
AFTER (the stopped replica, left stopped by the hook):
mtusi-lab4-api-2 host=4026e296947c exited exit=0 pid=0 started=2026-10-25T11:41:59.568
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api1 172.25.0.2 UP mtusi-lab4-api-1 X-Instance-Id=8691c30aa702
(other slots: MAINT, no container)
[exit 0, 0.8 s]
$ sh scripts/restore-api.sh
mtusi-lab4-api-1 Up 32 seconds (healthy)
mtusi-lab4-api-2 Up 6 seconds (healthy)
Container mtusi-lab4-api-1 Running
Container mtusi-lab4-api-2 Starting
Container mtusi-lab4-api-2 Started
Container mtusi-lab4-api-1 Waiting
Container mtusi-lab4-api-2 Waiting
Container mtusi-lab4-api-1 Healthy
Container mtusi-lab4-api-2 Healthy
[exit 0, 6.9 s]
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['4026e296947c', '8691c30aa702']
[exit 0, 1.5 s]
Replicas stopped by the public test across runs: ['mtusi-lab4-api-1', 'mtusi-lab4-api-2']
+32
View File
@@ -0,0 +1,32 @@
# make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3
# 2026-10-25T11:41:08Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов
Containers BEFORE:
mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=28760 started=2026-10-25T11:40:06.635
mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=28857 started=2026-10-25T11:40:07.085
mtusi-lab4-lb-1 host=1282a32916ca running exit=0 pid=29105 started=2026-10-25T11:40:13.103
mtusi-lab4-migrate-1 host=62f12280b06e exited exit=0 pid=0 started=2026-10-25T11:39:59.135
mtusi-lab4-postgres-1 host=286ea382ba46 running exit=0 pid=28093 started=2026-10-25T11:39:53.269
mtusi-lab4-s3-1 host=53564c9a38b4 running exit=0 pid=28181 started=2026-10-25T11:39:53.656
mtusi-lab4-s3-init-1 host=ab8ae8b24a67 exited exit=0 pid=0 started=2026-10-25T11:40:01.363
mtusi-lab4-valkey-1 host=44817703634f running exit=0 pid=28258 started=2026-10-25T11:39:53.966
$ make test-persistence ACTION_SCRIPT=scripts/restart.cmd
python tests/resilience.py --mode persistence --action-script "scripts/restart.cmd" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 61.5 s]
Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):
mtusi-lab4-api-1 host=8691c30aa702 running exit=0 pid=33790 started=2026-10-25T11:41:59.187
mtusi-lab4-api-2 host=4026e296947c running exit=0 pid=33848 started=2026-10-25T11:41:59.568
mtusi-lab4-lb-1 host=1282a32916ca running exit=0 pid=34447 started=2026-10-25T11:42:09.092
mtusi-lab4-migrate-1 host=62f12280b06e exited exit=0 pid=0 started=2026-10-25T11:42:01.016
mtusi-lab4-postgres-1 host=286ea382ba46 running exit=0 pid=32342 started=2026-10-25T11:41:15.594
mtusi-lab4-s3-1 host=53564c9a38b4 running exit=0 pid=32960 started=2026-10-25T11:41:35.722
mtusi-lab4-s3-init-1 host=ab8ae8b24a67 exited exit=0 pid=0 started=2026-10-25T11:42:03.276
mtusi-lab4-valkey-1 host=44817703634f running exit=0 pid=32296 started=2026-10-25T11:41:15.515
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.5 s: ['4026e296947c', '8691c30aa702']
[exit 0, 1.2 s]
+116
View File
@@ -0,0 +1,116 @@
# HAProxy 3.2 LTS — the single public entry point of the lab 4 stand (compose service `lb`).
#
# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the
# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and
# re-resolves every second, so scale up/down, a restarted container with a new IP and a
# full stack restart are picked up without restarting the LB. A stopped container drops
# out of DNS and its slot goes to MAINT.
#
# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey + S3 reachable, not
# draining) every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide
# anything on their own, and there are NO fallback servers without checks: a replica that
# answers /health/live but not /health/ready never gets traffic while a fully ready one exists.
#
# Degraded mode (lab 4): S3 is a SHARED dependency. If it is down, /health/ready is 503 on
# every replica and backend `api` has no server. Instead of answering 503 to everything, the
# frontend then switches to backend `api_core`: the same replicas, checked by
# /health/ready/core (PostgreSQL + Valkey, not draining). Chats and text messages keep working,
# every attachment request is answered 503 by the API itself (fast, S3_TIMEOUT_SECONDS).
# A replica that alone lost S3 (e.g. its storage network) is DOWN in `api` while the others are
# UP, so it gets no traffic at all.
# Worst case detection of a failed replica (a check answered at once counts at once, a check
# that hangs counts only after `timeout check`):
# fast explicit 503 (drain after SIGTERM): inter 1 s + fastinter 0.5 s ~= 1.5 s;
# slow ready (PostgreSQL/Valkey unreachable: the answer takes ~1 s = timeout check)
# or frozen process: inter 1 s + 1 s + fastinter 0.5 s + 1 s ~= 3.5 s.
# Requests that could not even be SENT (connection refused/reset before the request)
# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST,
# because the API has not seen them. Nothing that may have reached the API is retried by
# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key.
# No sticky sessions: sessions live in Valkey, every replica serves every user.
global
log stdout format raw local0 info
maxconn 4096
# Runtime API for `show servers state` / `show stat` (scripts and tests, no network port).
stats socket /var/lib/haproxy/admin.sock mode 600 level admin
hard-stop-after 15s
resolvers docker
nameserver dns1 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold other 1s
hold refused 1s
hold nx 1s
hold timeout 1s
hold obsolete 1s
defaults
mode http
log global
option httplog
option dontlognull
option http-keep-alive
timeout connect 1s
timeout client 30s
timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first
timeout http-request 10s
timeout http-keep-alive 10s
timeout queue 3s
timeout check 1s
retries 2
retry-on conn-failure
option redispatch 1
# Errors generated by the LB itself, in the contract error format with the same request id.
# They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the
# request is unknown (a replica died mid-request / did not answer in time): retry with the
# same Idempotency-Key. 503 = no ready replica at all.
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
frontend public
bind :8080
# One request id end to end: keep a valid client X-Request-Id, otherwise create one; the
# API echoes it, and errors generated by the LB itself carry the same value.
http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ }
http-request set-var(txn.request_id) req.hdr(x-request-id)
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\""
use_backend api if { nbsrv(api) gt 0 }
default_backend api_core
backend api
balance roundrobin
# Uploads (<= 10 MiB body + verification + S3 PUT): the API's own budget is
# UPLOAD_READ_TIMEOUT 30 s + IMAGE_VERIFY 10 s + S3; the LB must not cut it at 10 s (a 504
# after a successful upload would make the client upload a duplicate).
acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$
http-request set-timeout server 60s if is_upload METH_POST
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
server-template api 8 api:8080
# Degraded mode only (see the header): used when NO replica is fully ready.
backend api_core
balance roundrobin
acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$
http-request set-timeout server 60s if is_upload METH_POST
option httpchk
http-check send meth GET uri /health/ready/core ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
server-template core 8 api:8080
# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now.
frontend stats
bind :8404
stats enable
stats uri /
stats refresh 2s
stats show-legends
+45
View File
@@ -0,0 +1,45 @@
#!/bin/sh
# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume +
# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`.
#
# Identities (SeaweedFS s3.json, actions scoped to the bucket):
# bootstrap-admin Admin (create bucket) -> only s3-init has these keys
# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin)
# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing)
# There is NO anonymous identity, so every unsigned request is refused: the bucket is private.
#
# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on
# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the
# container network `storage` (internal, not published on the host). Telemetry is off.
set -eu
umask 077
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:?S3_BUCKET is required}"
cat > /tmp/s3.json <<EOF
{
"identities": [
{"name": "bootstrap-admin",
"credentials": [{"accessKey": "$(secret s3_admin_access_key_id)", "secretKey": "$(secret s3_admin_secret_access_key)"}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "messenger-api",
"credentials": [{"accessKey": "$(secret s3_api_access_key_id)", "secretKey": "$(secret s3_api_secret_access_key)"}],
"actions": ["Read:${bucket}", "Write:${bucket}"]},
{"name": "messenger-reconciler",
"credentials": [{"accessKey": "$(secret s3_reconciler_access_key_id)", "secretKey": "$(secret s3_reconciler_secret_access_key)"}],
"actions": ["Read:${bucket}", "Write:${bucket}", "List:${bucket}"]}
]
}
EOF
# Memory: by default SeaweedFS buffers every in-flight upload without a limit and the Go runtime
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (evidence/memory). The upload
# limits make excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set
# in compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
exec weed server \
-dir=/data \
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
-master.telemetry=false \
-master.volumeSizeLimitMB=1024 -volume.max=0 \
-filer -filer.disableDirListing -filer.concurrentUploadLimitMB=128 \
-volume.concurrentUploadLimitMB=128 \
-s3 -s3.ip.bind=0.0.0.0 -s3.port=8333 -s3.config=/tmp/s3.json -s3.concurrentUploadLimitMB=128 \
-s3.iam=false -s3.autoCreateBucket=false -s3.port.iceberg=0 -s3.port.lance=0
+17
View File
@@ -0,0 +1,17 @@
# Valkey is the single store of sessions (SESSION_STORE=valkey), so it must be durable.
# AOF with fsync on every write: a confirmed login or logout survives a crash/restart,
# a logout can never be lost and "resurrect" a token. Sessions are small and rare
# writes, so the fsync cost is acceptable here (see REPORT.md).
appendonly yes
appendfsync always
appenddirname "appendonlydir"
# RDB snapshots are not needed on top of AOF.
save ""
dir /data
port 6379
protected-mode yes
# Sessions are never evicted silently: if memory is full, writes fail and login returns 503.
maxmemory 128mb
maxmemory-policy noeviction
# Password, generated by scripts/init_secrets.py and written here at container start.
include /tmp/requirepass.conf
+65
View File
@@ -0,0 +1,65 @@
-- 0001: initial schema of the messenger (lab 2). Design rationale: docs/DATA_MODEL.md
-- Applied once by `python -m messenger.migrate`; the version is recorded in schema_migrations.
-- User-supplied free text (display_name, title, message text) is stored as UTF-8 in BYTEA:
-- the contract allows U+0000, and PostgreSQL TEXT cannot contain NUL bytes.
-- Lengths are limited in code points by the API; the CHECKs below are byte-level safety nets
-- (one code point is at most 4 bytes in UTF-8).
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name bytea NOT NULL,
password_hash text NOT NULL, -- Argon2id PHC string, never the password
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (octet_length(display_name) BETWEEN 1 AND 400)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title bytea NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
-- Per-chat clock: created_at of the newest message. Updating this row serialises writers
-- of one chat, so message timestamps strictly increase in commit order (see DATA_MODEL.md).
last_message_at timestamptz,
CONSTRAINT chats_title_len CHECK (octet_length(title) BETWEEN 1 AND 400)
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
position smallint NOT NULL, -- order of member_ids sent by the creator
chat_created_at timestamptz NOT NULL, -- copy of chats.created_at for keyset listing
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
);
-- GET /chats of one user: (user_id, created_at, id) range scan, no sort.
CREATE INDEX chat_members_user_keyset ON chat_members (user_id, chat_created_at, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text bytea NOT NULL,
created_at timestamptz NOT NULL,
-- The sender must be a member of this chat: enforced by the database, not only by the API.
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT messages_text_len CHECK (octet_length(text) BETWEEN 1 AND 16000)
);
-- History page: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
-- becomes an index range scan that starts exactly at the cursor.
CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id);
-- Sessions for SESSION_STORE=postgres (the default store is Valkey; see docs/DATA_MODEL.md).
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY, -- SHA-256 of the token; the token is not stored
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32)
);
CREATE INDEX sessions_expires_at ON sessions (expires_at);
+25
View File
@@ -0,0 +1,25 @@
-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages.
-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe
-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции").
--
-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key.
-- The method (POST) and the path template are fixed for this table, so the scope
-- is (user_id, chat_id, key). The key row, the message and the result are written in
-- ONE transaction (repo.post_message): there is no "pending" state that a crash could
-- leave behind, and the stored result is the message row itself.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
chat_id uuid NOT NULL REFERENCES chats (id),
key text NOT NULL,
request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body
-- Inserted before the message in the same transaction, hence checked at COMMIT.
message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, chat_id, key),
CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32),
CONSTRAINT idempotency_keys_message_key UNIQUE (message_id)
);
-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h).
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
+50
View File
@@ -0,0 +1,50 @@
-- 0003 (lab 3, track 1 improvement): the whole idempotent "send message" as ONE statement.
-- Additive (a new function), so replicas of the previous version keep working while it is
-- applied, and the previous write path (MESSAGE_WRITE_PATH=multi) stays usable for A/B runs.
--
-- Same semantics and the same SQL as repo.post_message (multi-statement path), but executed
-- inside the server: the API sends one call instead of ~11 round trips in two transactions
-- (membership check, BEGIN, key claim, chat clock, message, COMMIT, pool checks). Called in
-- autocommit mode, so the function call IS the transaction: key row, chat clock and message
-- commit together or not at all. VOLATILE + READ COMMITTED: every statement inside takes a
-- fresh snapshot, so after waiting on a concurrent claim of the same key the stored result
-- of the winner is visible, exactly as in the multi-statement path.
--
-- o_status: created | replayed | conflict | not_found | retry (key vanished in between: cleanup)
CREATE FUNCTION post_message(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
p_text bytea, p_ttl_seconds integer)
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz)
LANGUAGE plpgsql VOLATILE AS $$
DECLARE
v_created timestamptz;
BEGIN
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
IF NOT FOUND THEN
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz;
RETURN;
END IF;
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, p_message)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
IF FOUND THEN
-- Per-chat clock under the chat row lock (see migrations/0001 and docs/DATA_MODEL.md).
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES (p_message, p_chat, p_user, p_text, v_created);
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created;
RETURN;
END IF;
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
RETURN QUERY SELECT 'retry'::text, NULL::uuid, NULL::bytea, NULL::timestamptz;
END IF;
END
$$;
+145
View File
@@ -0,0 +1,145 @@
-- 0004 (lab 4): attachments. Metadata live here, the bytes in the private S3 bucket.
-- Additive (new tables, a relaxed CHECK, a NEW function post_message_v2), so replicas of the
-- lab 3 version keep working while it is applied (they never send attachment_ids).
--
-- Consistency between the two stores (docs/CONSISTENCY.md):
-- 1. INSERT the row as status='pending' (intent: the server-generated object_key is known
-- BEFORE anything is written to S3) and COMMIT;
-- 2. PUT the object into S3;
-- 3. UPDATE status='ready' WHERE status='pending'; only then 201.
-- Invariant: an object never exists without a row that names its key (row first, object
-- second; on cleanup object first, row second). A crash/outage between the steps leaves a
-- 'pending' row; the janitor deletes its object (idempotent) and then the row once it is older
-- than PENDING_GRACE_SECONDS. The reconciler (python -m messenger.reconcile) checks the
-- invariant over the whole bucket.
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
owner_id uuid NOT NULL,
filename text NOT NULL, -- display name only, never a path or a key
content_type text NOT NULL, -- detected from the bytes, not from the header
size_bytes integer NOT NULL,
sha256 bytea NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
-- Random server-generated key ("o/" + 128 random bits in hex), unrelated to id/filename.
-- COLLATE "C": byte order = S3 ListObjectsV2 order, the reconciler merge-joins both lists.
object_key text COLLATE "C" NOT NULL,
status text NOT NULL DEFAULT 'pending',
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
-- Set by the reconciler when the object of a READY row is missing or has the wrong size:
-- the row is quarantined (content -> 404, cannot be attached), never deleted, and the mark
-- is removed again if the object comes back (restore from backup).
lost_at timestamptz,
lost_reason text,
-- The uploader must be a member of the chat (enforced by the database too).
CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^o/[0-9a-f]{32}$'),
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32),
CONSTRAINT attachments_pixels CHECK (width >= 1 AND height >= 1 AND width::bigint * height <= 40000000),
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_lost CHECK ((lost_at IS NULL) = (lost_reason IS NULL) AND (lost_at IS NULL OR status = 'ready'))
);
-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads).
CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending';
-- attachment_ids of a message, twice on purpose (both written in the message's transaction):
-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the
-- keyset range scan of messages stays ONE index scan, without a per-row sub-query;
-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the
-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id.
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
position smallint NOT NULL,
attachment_id uuid NOT NULL REFERENCES attachments (id),
PRIMARY KEY (message_id, position),
CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id),
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9)
);
-- "Is this attachment used by any message?" (janitor/reconciler never touch used ones).
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
-- Lab 4: text may be "" when the message has attachments (the API enforces "text or files").
-- DROP CONSTRAINT takes a short exclusive lock on messages and the new CHECK scans the table
-- once, inside this migration's transaction (seconds for a lab-sized table; for a big table
-- this would be ADD ... NOT VALID + VALIDATE in a separate migration).
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (octet_length(text) <= 16000);
-- post_message_v2: post_message (0003) + attachment_ids, still ONE statement and ONE transaction.
-- The attachments are checked BEFORE the key is claimed and the key is claimed only if they are
-- fine, so a failed check never leaves a key behind. A replay (same key, same body) returns the
-- stored message even if an attachment was quarantined since: the result of a key is final.
--
-- o_status: created | replayed | conflict | not_found | retry | attachment_not_found | attachment_conflict
CREATE FUNCTION post_message_v2(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
LANGUAGE plpgsql VOLATILE AS $$
DECLARE
v_created timestamptz;
v_found integer;
v_usable integer;
v_check text := 'ok';
BEGIN
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
IF NOT FOUND THEN
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
RETURN;
END IF;
IF cardinality(p_attachments) > 0 THEN
-- Visible = ready attachment of THIS chat (pending, foreign or unknown -> 404).
-- Usable = visible and not quarantined (lost -> 409, like a failed attachment).
-- FOR SHARE: a concurrent quarantine waits until this message is committed.
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) INTO v_found, v_usable
FROM (SELECT a.lost_at FROM attachments a
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status = 'ready'
FOR SHARE) a;
IF v_found <> cardinality(p_attachments) THEN
v_check := 'attachment_not_found';
ELSIF v_usable <> v_found THEN
v_check := 'attachment_conflict';
END IF;
END IF;
IF v_check = 'ok' THEN
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, p_message)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
IF FOUND THEN
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
INSERT INTO message_attachments (message_id, position, attachment_id)
SELECT p_message, (a.ord - 1)::smallint, a.id
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
RETURN;
END IF;
END IF;
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
m.attachment_ids
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
END IF;
END
$$;
+6
View File
@@ -0,0 +1,6 @@
# Student tests (in-process, via Starlette TestClient). Runtime deps come from requirements.txt.
pytest==9.1.1
httpx2==2.13.1
jsonschema==4.26.0
# Lab 4 tests: generate PNG/JPEG fixtures (valid, corrupt, oversized) on the fly.
pillow==12.3.0
+20
View File
@@ -0,0 +1,20 @@
# Direct runtime dependencies. The pinned lock with hashes is requirements.txt:
# docker run --rm -v "$PWD:/w" -w /w python:3.13.16-slim-bookworm \
# sh -c "pip install -q pip-tools==7.6.1 && pip-compile --generate-hashes --strip-extras --no-emit-index-url -o requirements.txt requirements.in"
starlette==1.7.0
uvicorn==0.54.0
# Fast C parser/event loop used by uvicorn in the container (Linux only).
httptools==0.9.0
uvloop==0.23.0 ; sys_platform != "win32"
# PostgreSQL driver (libpq bundled in the binary wheel) and its async connection pool.
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
# Valkey client (Valkey speaks the Redis protocol).
redis==8.1.0
# Argon2id password KDF.
argon2-cffi==25.1.0
# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel).
aiobotocore==3.9.2
# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile).
+1356
View File
File diff suppressed because it is too large Load Diff
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
container states before/after every hook. Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
WINDOWS = os.name == "nt"
HOOK = ".cmd" if WINDOWS else ".sh"
PYTHON = sys.executable
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
MAKE_PY = "python" if WINDOWS else "python3"
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
ENV = {**os.environ, "PYTHONUTF8": "1"}
if WINDOWS:
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
def now() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
started = time.monotonic()
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, env={**ENV, **(env or {})})
took = time.monotonic() - started
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
class Log:
def __init__(self, name: str, title: str):
self.path = EVIDENCE / name
self.path.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8").stdout.strip()
self.text = f"# {title}\n# {now()}, commit {commit}\n"
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
code, out = sh(cmd, **kwargs)
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
self.save()
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
return code
def note(self, text: str) -> None:
self.text += f"\n{text}\n"
self.save()
def save(self) -> None:
self.path.write_text(self.text, encoding="utf-8")
def states() -> list[str]:
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
rows = []
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
st = item["State"]
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
return sorted(rows)
def main() -> int:
EVIDENCE.mkdir(exist_ok=True)
make = shutil.which("make", path=ENV["PATH"]) or "make"
# 1. Clean clone -> secrets -> build -> start
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
log.run(["git", "status", "--short", "--branch"])
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
if not (ROOT / ".env").exists():
shutil.copy(ROOT / ".env.example", ROOT / ".env")
log.note("$ cp .env.example .env")
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
# 2. make check / validate / test
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
stopped: set[str] = set()
import urllib.request
for attempt in range(1, 7):
# The public test stops the replica that served its first POST; its request sequence is
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
# (attempt - 1) extra requests so that the other replica becomes the target as well.
for _ in range(attempt - 1):
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
before = {r for r in states() if "-api-" in r and " running " in r}
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
after = states()
gone = [r for r in after if "-api-" in r and " exited " in r]
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
log.run([PYTHON, "scripts/lbctl.py", "status"])
stopped.update(r.split()[0] for r in gone)
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
shown="sh scripts/restore-api.sh")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
if len(stopped) >= 2:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
print("done:", EVIDENCE)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
request straight to S3 (from inside the storage network) is refused, the signed one works.
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import subprocess
import sys
import urllib.error
import urllib.request
import uuid
from email.message import Message
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
PASSWORD = "Demo-password-Ω-12345"
class Client:
def __init__(self, base: str):
self.base = base.rstrip("/")
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
headers: dict | None = None) -> tuple[int, Message, bytes]:
h = dict(headers or {})
if token:
h["Authorization"] = "Bearer " + token
data = raw
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
h["Content-Type"] = "application/json"
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, r.read() # case-insensitive HTTPMessage
except urllib.error.HTTPError as e:
return e.code, e.headers, e.read()
def show(label: str, status: int, headers: Message, payload: bytes, *, binary: bool = False) -> None:
inst = headers.get("X-Instance-Id", "-")
if binary:
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
else:
text = payload.decode("utf-8", "replace")[:300]
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
def user(c: Client, label: str) -> dict:
name = f"{label}_{uuid.uuid4().hex[:8]}"
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
"password": PASSWORD})
assert status == 201, body
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
session = json.loads(body)
return {"id": session["user"]["id"], "token": session["token"], "name": name}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
c = Client(args.base_url)
content = FIXTURE.read_bytes()
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
print(f"users: alice, bob (members), eve (outsider)")
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
chat = json.loads(body)
print("\n1. upload -> message -> download by the second member")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
raw=content, headers={"Content-Type": "image/png"})
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
att = json.loads(b)
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
show("GET /attachments/{id} (bob)", st, h, b)
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
print(f" SHA-256 of the download == fixture == metadata: {same}")
print("\n2. outsider and direct S3 access")
for suffix in ("", "/content"):
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
f"{key not in json.dumps(att)})")
probe = (
"import urllib.request as u, urllib.error as e, asyncio\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
"asyncio.run(m())\n")
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
capture_output=True, text=True).stdout
for line in out.splitlines():
print(" (inside network storage) " + line)
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
return 0 if same else 1
if __name__ == "__main__":
sys.exit(main())
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4).
python scripts/explain_history.py --output evidence/explain-history.txt
Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in
30 other chats, then explains the EXACT statements of src/messenger/repo.py:
1. first page (HISTORY_FIRST)
2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement
reused by the driver would get it
3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan
4. the rejected OFFSET pagination for the same late page
Runs psql inside the postgres container; no database port is published.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL)
CHAT = "00000000-0000-4000-8000-00000000e002"
USER = "00000000-0000-4000-8000-00000000e001"
TARGET = 20_000
LATE = 19_950 # cursor position: the 50 newest messages remain
SEED = f"""
\\set ON_ERROR_STOP on
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_by, created_at)
SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'),
'{USER}', timestamptz '2026-01-01 00:00:00+00'
FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM generate_series(1::int, {TARGET}::int) AS g
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}');
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g
WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}'
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
COMMIT;
ANALYZE messages;
"""
def pg(sql_param: str) -> str:
"""Turn psycopg %s placeholders into $1..$n for a PREPARE statement."""
parts = sql_param.split("%s")
return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts))
EXPLAIN = f"""
\\set ON_ERROR_STOP on
\\pset pager off
SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages;
SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}'
ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset
SET plan_cache_mode = force_generic_plan;
PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)};
PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)};
PREPARE or_page(uuid, timestamptz, uuid, int) AS
SELECT id, sender_id, text, created_at FROM messages
WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4;
PREPARE offset_page(uuid, int, int) AS
SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3;
\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51);
\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 4. REJECTED: OFFSET pagination, same late page'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51);
"""
def psql(script: str) -> str:
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"],
cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True)
return result.stdout
def run() -> str:
psql(SEED)
return psql(EXPLAIN)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--output", default=None)
args = parser.parse_args()
text = run()
print(text)
if args.output:
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
return 0
if __name__ == "__main__":
sys.exit(main())
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Create local synthetic credentials for the stand in ./secrets (ignored by Git).
python scripts/init_secrets.py
Idempotent: existing files are kept, so re-running never changes passwords of a
database that already has data. Prints only file names, never values.
Delete ./secrets together with the volumes (`docker compose down -v`) to start over.
"""
from __future__ import annotations
import os
import secrets
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
def write(name: str, value: str) -> None:
path = SECRETS / name
if path.exists():
print(f"keep secrets/{name}")
return
path.write_text(value + "\n", encoding="utf-8", newline="\n")
# Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the
# compose bind mount. Local synthetic lab credentials only.
os.chmod(path, 0o644)
print(f"created secrets/{name}")
def read(name: str) -> str:
return (SECRETS / name).read_text(encoding="utf-8").strip()
def main() -> int:
SECRETS.mkdir(exist_ok=True)
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
write("valkey_password", secrets.token_urlsafe(24))
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
# Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh):
# admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List.
for who in ("admin", "api", "reconciler"):
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
return 0
if __name__ == "__main__":
sys.exit(main())
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""What the load balancer thinks right now, mapped to real containers.
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
so no admin port is published. Container hostname = X-Instance-Id of that replica.
Prints no secrets. Used by student-tests and bench scripts as a library as well.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
def run(*args: str, timeout: float = 60) -> str:
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
timeout=timeout)
if result.returncode != 0:
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
return result.stdout
def replicas(all_states: bool = True) -> list[dict]:
"""API containers of this compose project (running and stopped)."""
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
if not ids:
return []
data = json.loads(run("docker", "inspect", *ids))
result = []
for item in data:
networks = item["NetworkSettings"]["Networks"] or {}
result.append({
"id": item["Id"][:12],
"name": item["Name"].lstrip("/"),
"hostname": item["Config"]["Hostname"],
"state": item["State"]["Status"],
"health": (item["State"].get("Health") or {}).get("Status", "-"),
"exit_code": item["State"]["ExitCode"],
"started_at": item["State"]["StartedAt"],
"pid": item["State"]["Pid"],
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
})
return sorted(result, key=lambda r: r["name"])
def lb_servers() -> list[dict]:
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
servers = []
for line in raw.splitlines():
parts = line.split()
if len(parts) < 7 or not parts[0].isdigit():
continue
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
if admin & ADMIN_RMAINT or addr == "-":
state = "MAINT"
else:
state = OP_STATE.get(op, str(op))
container = by_ip.get(addr)
servers.append({"slot": name, "addr": addr, "state": state,
"container": container["name"] if container else None,
"instance_id": container["hostname"] if container else None})
return servers
def up_instances() -> set[str]:
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
def wait_up(count: int, timeout: float = 60) -> float:
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
started = time.monotonic()
while True:
try:
if len(up_instances()) == count:
return time.monotonic() - started
except RuntimeError:
pass # lb restarting
if time.monotonic() - started > timeout:
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
time.sleep(0.25)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status")
sub.add_parser("replicas")
wait = sub.add_parser("wait-up")
wait.add_argument("count", type=int)
wait.add_argument("--timeout", type=float, default=60)
args = parser.parse_args()
if args.command == "status":
for s in lb_servers():
if s["state"] != "MAINT" or s["container"]:
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
print(f"(other slots: MAINT, no container)")
elif args.command == "replicas":
for r in replicas():
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
else:
took = wait_up(args.count, args.timeout)
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
return 0
if __name__ == "__main__":
sys.exit(main())
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash.
rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh"
exit /b %ERRORLEVEL%
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS).
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
# Output is suppressed by the test; run it by hand to see the steps.
set -eu
cd "$(dirname "$0")/.."
# 1. Entry point and API first: no request is served while the stores restart underneath.
docker compose stop lb api
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
# flush), then start again. The S3 bytes live on the named volume s3data.
docker compose restart postgres valkey s3
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
# the new replica addresses through Docker DNS by itself - no stale backend list.
docker compose start api
docker compose start lb
echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved"
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker
# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s);
# `python scripts/lbctl.py wait-up 2` waits for that explicitly.
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --no-recreate --no-deps --wait api
docker compose ps api --format '{{.Name}} {{.Status}}'
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
# 1. build the new image from the working tree;
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
# running old version: expand first, contract in a later release);
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
# the old replicas that were not replaced yet keep running.
set -eu
cd "$(dirname "$0")/.."
PY="${PYTHON:-$(command -v python3 || command -v python)}"
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
export BUILD_REVISION
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
old="$(docker compose ps -q api)"
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
docker compose build api
docker compose run --rm --no-deps migrate
for id in $old; do
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
docker stop --time 15 "$id" >/dev/null
docker rm "$id" >/dev/null
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
echo "rolling-update: replaced $name"
done
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
done
+7
View File
@@ -0,0 +1,7 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash;
rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment.
rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh"
exit /b %ERRORLEVEL%
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`.
# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname,
# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are
# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy);
# restore it explicitly afterwards: scripts/restore-api.sh.
# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s)
# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process
# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed).
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
mode="${STOP_MODE:-stop}"
target=""
# Only RUNNING containers of THIS compose project's service `api` are candidates.
for id in $(docker compose ps -q api); do
if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
target="$id"
fi
done
if [ -z "$target" ]; then
echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2
exit 3
fi
name="$(docker inspect -f '{{.Name}}' "$target")"
case "$mode" in
stop) docker stop --time 15 "$target" >/dev/null ;;
kill) docker stop --time 0 "$target" >/dev/null ;;
*) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;;
esac
echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")"
+3
View File
@@ -0,0 +1,3 @@
"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»)."""
__version__ = "1.0.0"
+88
View File
@@ -0,0 +1,88 @@
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy):
1. readiness flips to 503 (`app.state.draining = True`), requests are still served;
2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has
marked this replica DOWN and sends nothing new here;
3. uvicorn closes the listening socket; in-flight requests get up to
SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish;
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL.
A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey.
"""
from __future__ import annotations
import logging
import os
import sys
import threading
from types import FrameType
import uvicorn
from . import __version__, logging_setup
from .app import create_app
from .config import ConfigError, Settings
log = logging.getLogger("messenger")
class GracefulServer(uvicorn.Server):
"""uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop."""
def __init__(self, config: uvicorn.Config, drain_seconds: float):
super().__init__(config)
self.drain_seconds = drain_seconds
self.drain_timer: threading.Timer | None = None
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
self.config.app.state.draining = True # config.app is our Starlette instance
if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit:
log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds,
"timeout_s": self.config.timeout_graceful_shutdown}})
# uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe.
self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame))
self.drain_timer.daemon = True
self.drain_timer.start()
return
if self.drain_timer is not None:
self.drain_timer.cancel() # second signal: stop draining now
self._stop(sig, frame)
def _stop(self, sig: int, frame: FrameType | None) -> None:
if not self.should_exit:
log.info("shutdown_requested", extra={"fields": {"signal": sig}})
super().handle_exit(sig, frame)
def main() -> int:
try:
settings = Settings.from_env()
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
logging_setup.configure(settings.log_level)
app = create_app(settings)
config = uvicorn.Config(
app,
host=settings.host,
port=settings.port,
log_config=None,
access_log=False,
server_header=False,
proxy_headers=False,
lifespan="on",
timeout_graceful_shutdown=settings.shutdown_timeout,
)
server = GracefulServer(config, settings.drain_seconds)
log.info("starting", extra={"fields": {
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
}})
server.run()
log.info("stopped")
return 0
if __name__ == "__main__":
sys.exit(main())
+584
View File
@@ -0,0 +1,584 @@
"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
from __future__ import annotations
import asyncio
import contextlib
import hashlib
import json
import logging
import random
import re
import time
import uuid
from typing import Any
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import JSONResponse, Response, StreamingResponse
from starlette.routing import Route
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
from . import attachments, authhdr, cursor, docs, passwords, repo, validation
from .config import Settings
from .db import Database
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
from .sessions import SessionStore, build_store
from .storage import ObjectMissing, Storage
from .timefmt import rfc3339
log = logging.getLogger("messenger")
access_log = logging.getLogger("messenger.access")
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
# --------------------------------------------------------------------------
# Responses
# --------------------------------------------------------------------------
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
body = {"error": {"code": code, "message": message, "request_id": request_id}}
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
assert isinstance(exc, ApiError)
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Unknown routes and wrong methods also answer in the contract error format."""
assert isinstance(exc, HTTPException)
if exc.status_code == 404:
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
# --------------------------------------------------------------------------
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
# --------------------------------------------------------------------------
class RequestContextMiddleware:
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
included), writes one access-log line, turns crashes into 500."""
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id.encode("ascii")
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = request_id
status = 500
started = False
begin = time.perf_counter()
async def send_with_id(message: AsgiMessage) -> None:
nonlocal status, started
if message["type"] == "http.response.start":
started = True
status = message["status"]
headers = [(k, v) for k, v in message.get("headers", [])
if k.lower() not in (b"x-request-id", b"x-instance-id")]
headers.append((b"x-request-id", request_id.encode("latin-1")))
headers.append((b"x-instance-id", self.instance_id))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, send_with_id)
except asyncio.CancelledError:
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
if not started:
status = 503
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
await response(scope, receive, send_with_id)
raise
except Exception:
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
if not started:
status = 500
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
await response(scope, receive, send_with_id)
finally:
access_log.info("request", extra={"fields": {
"request_id": request_id,
"method": scope["method"],
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
"status": status,
"instance_id": self.instance_id.decode("ascii"),
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
}})
# --------------------------------------------------------------------------
# Request helpers
# --------------------------------------------------------------------------
def db_of(request: Request) -> Database:
return request.app.state.db
def sessions_of(request: Request) -> SessionStore:
return request.app.state.sessions
def storage_of(request: Request) -> Storage:
return request.app.state.storage
async def authenticate(request: Request) -> str:
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
Store failures are 503 (Unavailable), never 401 and never anonymous access.
"""
token = authhdr.parse_bearer(request.headers.get("authorization"))
user_id = await sessions_of(request).user_id_for(token)
if user_id is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return user_id
def path_uuid(request: Request, name: str) -> str:
return validation.parse_uuid(request.path_params[name], name)
async def read_json(request: Request) -> dict[str, Any]:
content_type = request.headers.get("content-type", "")
media_type, _, params = content_type.partition(";")
if media_type.strip().lower() != "application/json":
raise InvalidRequest("Content-Type должен быть application/json")
charset = params.strip().lower()
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
raise InvalidRequest("Поддерживается только кодировка UTF-8")
limit: int = request.app.state.settings.max_body_bytes
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > limit:
raise PayloadTooLarge()
chunks: list[bytes] = []
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise PayloadTooLarge()
chunks.append(chunk)
return validation.parse_json_object(b"".join(chunks))
def page_params(request: Request) -> tuple[int, str | None]:
query = request.query_params.multi_items()
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
raw_cursor = validation.single_query_param(query, "cursor")
if raw_cursor is not None and raw_cursor == "":
raise InvalidRequest("cursor не может быть пустым")
return limit, raw_cursor
async def reject_body(request: Request) -> None:
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
async for chunk in request.stream():
if chunk:
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
def idempotency_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
return validation.idempotency_key(values[0] if len(values) == 1 else None)
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
def ok(body: Any, status: int = 200) -> JSONResponse:
return JSONResponse(body, status_code=status)
# --------------------------------------------------------------------------
# Handlers
# --------------------------------------------------------------------------
async def live(request: Request) -> JSONResponse:
return ok({"status": "ok"})
async def ready(request: Request) -> JSONResponse:
"""200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey
(SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments)
answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(),
storage_of(request).ping())
if not db_ok:
raise Unavailable("PostgreSQL недоступен")
if not store_ok:
raise Unavailable("Хранилище сессий недоступно")
if not s3_ok:
raise Unavailable("Хранилище файлов (S3) недоступно")
return ok({"status": "ok"})
async def ready_core(request: Request) -> JSONResponse:
"""Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only.
If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to
replicas that are at least core-ready, so chats and messages keep working while every
attachment request answers 503 itself (see infra/haproxy/haproxy.cfg)."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
if not (db_ok and store_ok):
raise Unavailable("PostgreSQL или хранилище сессий недоступно")
return ok({"status": "ok"})
async def register_user(request: Request) -> JSONResponse:
data = await read_json(request)
validation.check_fields(data, {"username", "display_name", "password"})
username = validation.username_field(data)
display_name = validation.text_field(data, "display_name", 100)
password = validation.password_field(data)
password_hash = await passwords.hash_password(password)
user = await repo.create_user(db_of(request), username, display_name, password_hash)
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok(user.to_json(), 201)
async def create_session(request: Request) -> JSONResponse:
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
username, password = authhdr.parse_basic(request.headers.get("authorization"))
await reject_body(request)
found = await repo.get_credentials(db_of(request), username)
user, stored_hash = found if found else (None, None)
if not await passwords.verify_password(stored_hash, password) or user is None:
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
ttl = request.app.state.settings.session_ttl_seconds
token, expires_at = await sessions_of(request).create(user.id, ttl)
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
async def revoke_session(request: Request) -> Response:
token = authhdr.parse_bearer(request.headers.get("authorization"))
if not await sessions_of(request).revoke(token):
raise Unauthorized("Сессия недействительна, истекла или отозвана")
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
return Response(status_code=204)
async def me(request: Request) -> JSONResponse:
user_id = await authenticate(request)
user = await repo.get_user(db_of(request), user_id)
if user is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return ok(user.to_json())
async def create_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
data = await read_json(request)
validation.check_fields(data, {"title", "member_ids"})
title = validation.text_field(data, "title", 100)
member_ids = validation.member_ids_field(data)
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
return ok(chat.to_json(), 201)
async def list_chats(request: Request) -> JSONResponse:
user_id = await authenticate(request)
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
async def get_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
"""`text` is required and a string (also with attachments); `attachment_ids`, if present, an
array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments."""
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
attachment_ids = validation.attachment_ids_field(data)
text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1,
allow_blank=bool(attachment_ids))
return text, attachment_ids
async def create_message(request: Request) -> JSONResponse:
"""Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body
-> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session."""
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
settings = request.app.state.settings
if settings.message_write_path == "multi":
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds, attachment_ids)
else:
try:
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
except ApiError:
# Rare error path only: an outsider must still get 404, not a hint about the body.
await repo.ensure_member(db, user_id, chat_id)
raise
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
result = await repo.post_message_single_call(db, user_id, chat_id, key,
request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds, attachment_ids)
if result.replayed:
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
"message_id": result.message.id}})
return ok(result.message.to_json(), 201)
async def list_messages(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
items, has_more = await repo.list_messages(db, chat_id, after, limit)
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
# ------------------------------------------------------------------ attachments (lab 4)
def fault(request: Request, point: str) -> None:
"""Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only)."""
if point in request.app.state.settings.fault_points:
log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}})
raise Unavailable("Внедрённый отказ: " + point)
async def upload_attachment(request: Request) -> JSONResponse:
"""POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py)."""
settings: Settings = request.app.state.settings
reader = attachments.BodyReader(request)
slots: asyncio.Semaphore = request.app.state.upload_slots
acquired = False
try:
try:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
declared = attachments.declared_type(request)
length = attachments.declared_length(request, settings.max_upload_bytes)
try:
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
except asyncio.TimeoutError:
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
acquired = True
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length)
except ApiError:
# Early rejection: drop (not store) a bounded rest of the body so the client reads our error.
await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0)
raise
# The slot is held until the buffer is no longer needed (after the S3 PUT), so at most
# UPLOAD_CONCURRENCY bodies of this replica exist in memory at any moment.
return await store_upload(request, db, chat_id, user_id, filename, upload)
finally:
if acquired:
slots.release()
async def store_upload(request: Request, db: Database, chat_id: str, user_id: str, filename: str,
upload: attachments.Upload) -> JSONResponse:
"""intent (pending) -> S3 PUT -> ready -> 201 (order and failure analysis: docs/CONSISTENCY.md)."""
attachment_id = uuid.uuid4()
object_key = attachments.new_object_key()
storage = storage_of(request)
await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data),
upload.sha256, object_key)
await storage.put(object_key, upload.data, upload.info.content_type,
{"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()})
fault(request, "after_s3_put")
attachment = await attachments.mark_ready(db, attachment_id)
if attachment is None:
raise Unavailable("Загрузка не завершена, повторите запрос")
fault(request, "after_ready_commit")
log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id, "chat_id": chat_id,
"size": attachment.size_bytes, "type": attachment.content_type}})
return ok(attachment.to_json(), 201)
async def get_attachment(request: Request) -> JSONResponse:
user_id = await authenticate(request)
attachment_id = path_uuid(request, "attachment_id")
return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json())
def content_disposition(filename: str) -> str:
"""RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted."""
from urllib.parse import quote
fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename)
return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}"
async def download_attachment(request: Request) -> Response:
"""Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from
S3 in 64 KiB chunks (memory per download does not depend on the file size)."""
user_id = await authenticate(request)
attachment_id = path_uuid(request, "attachment_id")
query = validation.query_items(request.scope["query_string"])
try:
validation.variant_param(query)
except validation.NotFoundVariant:
await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first
raise NotFound("Такого варианта у вложения нет") from None
attachment = await attachments.get_visible(db_of(request), user_id, attachment_id)
if attachment.lost:
raise NotFound("Оригинал вложения утрачен")
try:
size, chunks = await storage_of(request).open_stream(attachment.object_key)
except ObjectMissing:
# A ready row without its object: data loss, reported by the reconciler (quarantine).
log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id}})
raise NotFound("Оригинал вложения утрачен") from None
if size != attachment.size_bytes:
log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id}})
await chunks.aclose()
raise Unavailable("Оригинал вложения временно недоступен")
headers = {
"Content-Length": str(size),
"Content-Disposition": content_disposition(attachment.filename),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
"ETag": '"' + attachment.sha256 + '"',
}
return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers)
def create_app(settings: Settings | None = None) -> Starlette:
settings = settings or Settings.from_env()
db = Database(settings)
sessions = build_store(settings, db)
storage = Storage(settings)
async def purge_loop() -> None:
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
while True:
await asyncio.sleep(600)
try:
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
if deleted:
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
except ApiError:
pass # database unavailable right now; next round
async def janitor_loop(every: float) -> None:
"""Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py).
Every replica runs it; the advisory lock lets one round run at a time."""
while True:
await asyncio.sleep(every * (0.5 + random.random()))
try:
removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds)
if removed:
log.info("stale_uploads_removed", extra={"fields": {"count": removed}})
except ApiError:
pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round
stats_log = logging.getLogger("messenger.stats")
async def stats_loop(every: float) -> None:
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
cpu = time.process_time()
db.pool.pop_stats()
while True:
await asyncio.sleep(every)
stats = db.pool.pop_stats()
now_cpu = time.process_time()
stats_log.info("pool_stats", extra={"fields": {
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
cpu = now_cpu
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
passwords.configure(settings.hash_concurrency)
await db.open()
await storage.open()
log.info("dependencies_configured", extra={"fields": {
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
"upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency,
"pending_grace_s": settings.pending_grace_seconds,
"fault_points": sorted(settings.fault_points)}})
tasks = [asyncio.create_task(purge_loop())]
if settings.janitor_interval_seconds > 0:
tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds)))
if settings.stats_log_seconds > 0:
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
try:
yield
finally:
for task in tasks:
task.cancel()
await sessions.close()
await storage.close()
await db.close()
routes = [
Route("/health/live", live, methods=["GET"]),
Route("/health/ready", ready, methods=["GET"]),
Route("/health/ready/core", ready_core, methods=["GET"]),
Route("/api/v1/users", register_user, methods=["POST"]),
Route("/api/v1/users/me", me, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]),
Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]),
Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]),
]
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
routes += [
Route("/docs", docs.swagger_page, methods=["GET"]),
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
]
app = Starlette(
routes=routes,
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
lifespan=lifespan,
)
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
app.state.settings = settings
app.state.db = db
app.state.sessions = sessions
app.state.storage = storage
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency)
app.state.draining = False
return app
+300
View File
@@ -0,0 +1,300 @@
"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket.
Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order:
auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413)
-> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After)
-> body in chunks: size limit, read deadline, format + pixel count from the first bytes
-> full integrity check (images.verify) -> SHA-256 / size known
-> INSERT row status='pending' with a fresh random object_key, COMMIT (intent)
-> PUT object to S3 (Content-MD5)
-> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready.
Why 201 is honest: it is sent only after BOTH the object and the ready row are durable.
What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md):
* before the intent COMMIT: nothing (no object was written);
* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the
object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the
object - idempotent, a missing key is fine - and then the row, once it is older than
PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of
COMMIT we do not know whether the row is ready, and deleting the object could destroy a
ready attachment. Only the janitor decides, under a row lock, from the committed state;
* after the ready COMMIT (answer lost): a correct ready attachment the client does not know.
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import logging
import secrets
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from starlette.requests import Request
from . import images
from .config import Settings
from .db import Database
from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
from .storage import OBJECT_PREFIX, Storage
from .timefmt import rfc3339
log = logging.getLogger("messenger.attachments")
JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas
@dataclass(frozen=True, slots=True)
class Attachment:
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
object_key: str
status: str
created_at: dt.datetime
updated_at: dt.datetime
lost: bool = False
def to_json(self) -> dict:
"""Public representation: no object key, no bucket, no internal state."""
return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename,
"content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256,
"status": "ready", "variants": [], "error_code": None,
"created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)}
COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, "
"a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL")
def _attachment(row) -> Attachment:
return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7],
row[8], row[9], row[10], row[11])
def new_object_key() -> str:
"""Server-generated, 128 random bits, unrelated to the attachment id and the filename."""
return OBJECT_PREFIX + secrets.token_hex(16)
# ------------------------------------------------------------------ SQL
async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str,
info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')",
(attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height,
object_key))
try:
await db.run(work, transaction=False)
except pg_errors.ForeignKeyViolation:
raise NotFound("Чат не найден") from None
async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None:
"""pending -> ready. None if the row is no longer pending (the janitor removed a stale intent)."""
async def work(conn: psycopg.AsyncConnection) -> Attachment | None:
cur = await conn.execute(
"UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() "
f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,))
row = await cur.fetchone()
return _attachment(row) if row else None
return await db.run(work, transaction=False)
async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment:
"""A READY attachment of a chat the user is a member of; anything else is 404 (unknown,
foreign chat, unfinished upload) - existence of foreign objects is never revealed."""
async def work(conn: psycopg.AsyncConnection) -> Attachment:
cur = await conn.execute(
f"SELECT {COLUMNS} FROM attachments a "
"JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s "
"WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Вложение не найдено")
return _attachment(row)
return await db.run(work, transaction=False)
async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int:
"""Janitor round: remove intents of uploads that never became ready.
Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in
mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object.
Order: object first (idempotent), row second; a crash in between leaves the pending row and
the next round repeats. Only status='pending' rows older than the grace period are touched:
fresh uploads and ready/used attachments are never selected.
"""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'pending' "
"AND created_at < clock_timestamp() - make_interval(secs => %s) "
"ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch))
rows = await cur.fetchall()
if not rows:
return 0
await asyncio.gather(*(storage.delete(key) for _, key in rows))
await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'",
([r[0] for r in rows],))
return len(rows)
return await db.run(work, deadline=storage.timeout + 5)
# ------------------------------------------------------------------ request body
class Busy(Unavailable):
"""All upload slots of this replica are taken: retry shortly (no body was read)."""
@property
def headers(self) -> dict[str, str]:
return {"Retry-After": "1"}
@dataclass
class Upload:
data: bytearray
info: images.ImageInfo
sha256: bytes
class BodyReader:
"""Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it
is consumed and can discard the rest after an early rejection."""
def __init__(self, request: Request):
self.receive = request.receive
self.done = False
async def chunk(self, timeout: float) -> bytes | None:
if self.done:
return None
message = await asyncio.wait_for(self.receive(), timeout)
if message["type"] == "http.disconnect":
self.done = True
raise InvalidRequest("Клиент прервал загрузку")
if not message.get("more_body", False):
self.done = True
return message.get("body", b"")
async def discard(self, limit: int, timeout: float) -> None:
"""After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that
writes the whole body before reading the answer still gets our JSON error instead of a
connection reset. Nothing is kept in memory; beyond the limit the connection is dropped."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
seen = 0
try:
while not self.done and seen <= limit:
remaining = deadline - loop.time()
if remaining <= 0:
return
seen += len(await self.chunk(remaining) or b"")
except (ApiError, asyncio.TimeoutError):
return
def declared_type(request: Request) -> str:
"""Content-Type of the upload must be image/png or image/jpeg (parameters ignored)."""
media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower()
if media_type not in (images.PNG, images.JPEG):
raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg")
return media_type
def declared_length(request: Request, limit: int) -> int | None:
raw = request.headers.get("content-length")
if raw is None:
return None # chunked transfer: the limit is enforced while reading
if not raw.isdigit():
raise InvalidRequest("Неверный Content-Length")
length = int(raw)
if length > limit:
raise PayloadTooLarge(f"Файл больше {limit} байт")
if length == 0:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
return length
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
length: int | None) -> Upload:
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
reading/decoding the rest."""
loop = asyncio.get_running_loop()
deadline = loop.time() + settings.upload_read_timeout_seconds
limit = settings.max_upload_bytes
buffer = bytearray(length) if length else bytearray()
view = memoryview(buffer) if length else None
filled = 0
digest = hashlib.sha256()
info: images.ImageInfo | None = None
while True:
remaining = deadline - loop.time()
if remaining <= 0:
raise InvalidRequest("Тело запроса не получено вовремя")
try:
chunk = await reader.chunk(remaining)
except asyncio.TimeoutError:
raise InvalidRequest("Тело запроса не получено вовремя") from None
if chunk is None:
break
if filled + len(chunk) > (length or limit):
raise PayloadTooLarge(f"Файл больше {limit} байт")
if view is not None:
view[filled:filled + len(chunk)] = chunk
else:
buffer += chunk
filled += len(chunk)
digest.update(chunk)
if info is None and filled:
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
if view is not None:
view.release()
if not filled:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
if length and filled != length:
raise InvalidRequest("Тело запроса короче Content-Length")
if info is None:
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
try:
async with verify_slots:
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
except images.ImageError as exc:
raise _image_error(exc) from None
except asyncio.TimeoutError:
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
except (images.VerifierUnavailable, MemoryError):
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
return Upload(buffer, info, digest.digest())
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
try:
info = images.sniff(buffer, settings.max_image_pixels)
except images.ImageError as exc:
raise _image_error(exc) from None
if info is not None and info.content_type != declared:
# "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa).
raise UnsupportedMediaType("Содержимое не совпадает с Content-Type")
return info
def _image_error(exc: images.ImageError) -> ApiError:
return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)
+48
View File
@@ -0,0 +1,48 @@
"""Parsing of the Authorization header. Any malformed value is 401, never 500.
Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs
Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the
local synthetic lab stand (lab 7 adds TLS).
"""
from __future__ import annotations
import base64
import binascii
import re
from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized
from .validation import USERNAME_RE
B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}")
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters
def _split(header: str | None) -> tuple[str, str]:
if not header:
return "", ""
scheme, _, value = header.strip().partition(" ")
return scheme.lower(), value.strip()
def parse_basic(header: str | None) -> tuple[str, str]:
"""(username, password) from `Basic base64(username:password)`; the password may contain ':'."""
invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
scheme, value = _split(header)
if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024:
raise invalid
try:
decoded = base64.b64decode(value, validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise invalid from None
username, sep, password = decoded.partition(":") # split at the FIRST colon
if not sep or USERNAME_RE.fullmatch(username) is None or not password:
raise invalid
return username, password
def parse_bearer(header: str | None) -> str:
scheme, value = _split(header)
if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None:
# Missing header, Basic on a business endpoint, garbage token: one answer.
raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE)
return value
+236
View File
@@ -0,0 +1,236 @@
"""Runtime configuration read from environment variables (12-factor style).
Non-secret settings have safe defaults. Connection strings with passwords are
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
Invalid values stop the process at startup with a clear message.
"""
from __future__ import annotations
import os
import re
import socket
from dataclasses import dataclass
from pathlib import Path
class ConfigError(ValueError):
pass
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
def _instance_id() -> str:
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
Docker sets the hostname of a container to its short container ID, so the value
is unique per replica, stable for the life of the process and maps 1:1 to a real
container (`docker inspect`), which scripts/stop-one.sh relies on.
"""
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
if INSTANCE_ID_RE.fullmatch(value) is None:
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
return value
def _int(name: str, default: int, low: int, high: int) -> int:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = int(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _float(name: str, default: float, low: float, high: float) -> float:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = float(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _bool(name: str, default: bool) -> bool:
raw = os.environ.get(name, "").strip().lower()
if raw == "":
return default
if raw not in {"true", "false", "1", "0"}:
raise ConfigError(f"{name} must be true/false, got {raw!r}")
return raw in {"true", "1"}
def _choice(name: str, default: str, allowed: set[str]) -> str:
value = os.environ.get(name, "").strip().lower() or default
if value not in allowed:
raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}")
return value
def _secret(name: str, required: bool) -> str:
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
path = os.environ.get(f"{name}_FILE", "").strip()
if path:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError as exc:
# The message names the file, never its content.
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
else:
value = os.environ.get(name, "").strip()
if required and not value:
raise ConfigError(f"{name}_FILE (or {name}) must be set")
return value
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
log_level: str
shutdown_timeout: float
max_body_bytes: int
docs_enabled: bool = True
database_url: str = ""
session_store: str = "postgres"
cache_url: str = ""
session_ttl_seconds: int = 3600
db_pool_min: int = 1
db_pool_max: int = 10
db_timeout_seconds: float = 3.0
cache_timeout_seconds: float = 0.5
hash_concurrency: int = 2
instance_id: str = "local"
drain_seconds: float = 0.0
idempotency_ttl_seconds: int = 86400
stats_log_seconds: float = 0.0
message_write_path: str = "function"
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
s3_endpoint: str = "http://s3:8333"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key_id: str = ""
s3_secret_access_key: str = ""
s3_timeout_seconds: float = 3.0
s3_connect_timeout_seconds: float = 1.0
s3_max_connections: int = 20
max_upload_bytes: int = 10 * 1024 * 1024
max_image_pixels: int = 40_000_000
upload_concurrency: int = 4
upload_queue_timeout_seconds: float = 5.0
upload_read_timeout_seconds: float = 30.0
image_verify_timeout_seconds: float = 10.0
image_verify_concurrency: int = 1
jpeg_max_memory_mb: int = 256
pending_grace_seconds: int = 900
janitor_interval_seconds: float = 60.0
fault_points: frozenset[str] = frozenset()
@property
def db_statement_timeout_ms(self) -> int:
# The server stops a statement a bit before the client-side deadline fires.
return max(100, int(self.db_timeout_seconds * 1000) - 500)
@classmethod
def from_env(cls) -> "Settings":
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
if log_level not in {"debug", "info", "warning", "error"}:
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
if session_store == "redis":
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
if session_store not in {"postgres", "valkey"}:
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
if db_pool_min > db_pool_max:
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
return cls(
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
port=_int("APP_PORT", 8080, 1, 65535),
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
log_level=log_level,
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
docs_enabled=_bool("DOCS_ENABLED", True),
database_url=_secret("DATABASE_URL", required=True),
session_store=session_store,
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
# Contract name. Minimum 1 s so the expiry test can use 2 s.
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
db_pool_min=db_pool_min,
db_pool_max=db_pool_max,
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
instance_id=_instance_id(),
# After SIGTERM: readiness is 503 but requests are still served for this long,
# so the load balancer stops routing here before the listener closes.
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
# Contract: a replay within 24 h returns the original result.
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
# multi-statement transaction; both have identical semantics (A/B on one image).
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
**s3_settings(),
)
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs.
FAULT_POINTS = {"after_s3_put", "after_ready_commit"}
def s3_settings(*, credentials_required: bool = True) -> dict:
"""S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys."""
faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip())
if faults - FAULT_POINTS:
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments"
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}")
return dict(
s3_endpoint=endpoint,
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
s3_bucket=bucket,
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
# An intent row still `pending` after this long belongs to a failed upload: the janitor
# deletes its object and the row. Far above the longest possible upload (~45 s).
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
fault_points=faults,
)
+56
View File
@@ -0,0 +1,56 @@
"""Opaque keyset cursors.
A cursor encodes the sort key `(created_at, id)` of the last item on the page
plus the resource it belongs to, e.g. `v1|m|<chat_id>|<ts_us>|<message_id>`,
packed in URL-safe base64. The client treats it as an opaque string; the server
rejects cursors of another chat/user (400) and any non-canonical encoding.
"""
from __future__ import annotations
import base64
import re
from .errors import InvalidRequest
from .validation import UUID_RE
B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}")
TS_RE = re.compile(r"[0-9]{1,19}")
# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z.
MAX_TS_US = 253402300799999999
MESSAGES = "m"
CHATS = "c"
SortKey = tuple[int, str]
def encode(scope: str, owner_id: str, key: SortKey) -> str:
ts, item_id = key
raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def decode(cursor: str, scope: str, owner_id: str) -> SortKey:
"""Return the sort key stored in the cursor or raise InvalidRequest."""
bad = InvalidRequest("Неверный cursor")
if B64_RE.fullmatch(cursor) is None:
raise bad
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii")
except (ValueError, UnicodeDecodeError):
raise bad from None
parts = raw.split("|")
if len(parts) != 5 or parts[0] != "v1":
raise bad
_, got_scope, got_owner, ts, item_id = parts
if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower():
raise bad
key = (int(ts), item_id)
if key[0] > MAX_TS_US:
raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL
if encode(got_scope, got_owner, key) != cursor:
raise bad # only the canonical form produced by the server is accepted
if got_scope != scope or got_owner != owner_id:
raise InvalidRequest("cursor относится к другому ресурсу")
return key
+131
View File
@@ -0,0 +1,131 @@
"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work.
Three independent limits keep a request from hanging when the database misbehaves:
* pool `timeout` — waiting for a free connection (pool exhausted);
* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow
statement or a statement stuck behind a lock;
* client deadline — covers what the server cannot report, e.g. a
paused/frozen PostgreSQL container or a dead network.
The client deadline needs care. Every unit of work runs as its own task; at the deadline
the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
psycopg query makes psycopg send a cancel request, and libpq starts that connection
(PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
The abandoned task instead ends by itself, bounded by:
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
breaks the socket within seconds, the connection is discarded by the pool;
* `statement_timeout` / `lock_timeout` on a live but slow server;
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
at most the pool timeout and gets 503.
Any failure becomes 503 `unavailable` — never 401, never a hang.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from typing import TypeVar
import psycopg
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.db")
T = TypeVar("T")
Work = Callable[[psycopg.AsyncConnection], Awaitable[T]]
# Errors that mean "the database is unavailable right now", not "the request is wrong".
# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown
# (server restart) are subclasses of OperationalError.
UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError)
def _forget(task: asyncio.Task) -> None:
"""Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'."""
if not task.cancelled():
task.exception()
class Database:
def __init__(self, settings: Settings):
self.deadline = settings.db_timeout_seconds
options = (
f"-c statement_timeout={settings.db_statement_timeout_ms} "
f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} "
"-c idle_in_transaction_session_timeout=10000"
)
self.pool = AsyncConnectionPool(
conninfo=settings.database_url,
min_size=settings.db_pool_min,
max_size=settings.db_pool_max,
timeout=settings.db_timeout_seconds, # max wait for a free connection
open=False,
name="messenger",
check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart
kwargs={
"options": options,
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
"application_name": "messenger-api",
# Autocommit connections: a single statement (post_message function, ping) is its
# own transaction with ONE round trip; multi-statement work uses an explicit
# conn.transaction() (BEGIN ... COMMIT), see run().
"autocommit": True,
# Detect a dead peer / lost network without any cancel request (see module doc).
"keepalives": 1,
"keepalives_idle": 5,
"keepalives_interval": 1,
"keepalives_count": 3,
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
},
)
async def open(self) -> None:
# wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet;
# readiness stays 503 until the database answers.
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close(timeout=2)
async def run(self, work: Work[T], deadline: float | None = None, *, transaction: bool = True) -> T:
"""Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception).
transaction=False: `work` issues exactly ONE statement, which autocommit makes atomic.
Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged;
database unavailability and the deadline become Unavailable (503).
"""
async def unit() -> T:
async with self.pool.connection() as conn:
if not transaction:
return await work(conn)
async with conn.transaction():
return await work(conn)
task = asyncio.ensure_future(unit())
try:
return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline)
except UNAVAILABLE_ERRORS as exc:
log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("База данных временно недоступна") from None
finally:
if not task.done():
# Deadline or cancelled request: stop waiting now. The task is left to finish
# on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
task.add_done_callback(_forget)
async def ping(self, timeout: float = 1.0) -> bool:
async def select_one(conn: psycopg.AsyncConnection) -> None:
await conn.execute("SELECT 1")
try:
await self.run(select_one, deadline=timeout, transaction=False)
return True
except Unavailable:
return False
+57
View File
@@ -0,0 +1,57 @@
"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs
The page shows the course contract contracts/openapi.json as is (it is not
generated from our code). Only `servers` is replaced with "/" so that the
"Try it out" requests go to the same host the page was opened from.
Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN.
Disable with DOCS_ENABLED=false.
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from starlette.requests import Request
from starlette.responses import HTMLResponse, JSONResponse
SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json"
SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0"
PAGE = f"""<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<title>Messenger lab1 — Swagger UI</title>
<link rel="stylesheet" href="{SWAGGER_UI}/swagger-ui.css">
</head>
<body>
<div id="swagger-ui"></div>
<script src="{SWAGGER_UI}/swagger-ui-bundle.js"></script>
<script>
window.ui = SwaggerUIBundle({{
url: "/openapi.json",
dom_id: "#swagger-ui",
tryItOutEnabled: true, // fields are editable right away
persistAuthorization: true, // X-User-Id from "Authorize" survives page reload
displayRequestDuration: true,
}});
</script>
</body>
</html>
"""
@lru_cache(maxsize=1)
def _spec() -> dict:
spec = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
spec["servers"] = [{"url": "/", "description": "этот сервер"}]
return spec
async def swagger_page(request: Request) -> HTMLResponse:
return HTMLResponse(PAGE)
async def openapi_json(request: Request) -> JSONResponse:
return JSONResponse(_spec())
+71
View File
@@ -0,0 +1,71 @@
"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md)."""
from __future__ import annotations
BEARER_CHALLENGE = 'Bearer realm="messenger"'
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
class ApiError(Exception):
status = 500
code = "internal_error"
default_message = "Внутренняя ошибка сервера"
def __init__(self, message: str | None = None):
self.message = message or self.default_message
super().__init__(self.message)
@property
def headers(self) -> dict[str, str]:
return {}
class InvalidRequest(ApiError):
status = 400
code = "invalid_request"
default_message = "Неверный запрос"
class Unauthorized(ApiError):
"""401 with the challenge of the scheme the endpoint expects (RFC 7235)."""
status = 401
code = "unauthorized"
default_message = "Требуется действительная сессия"
def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE):
super().__init__(message)
self.challenge = challenge
@property
def headers(self) -> dict[str, str]:
return {"WWW-Authenticate": self.challenge}
class NotFound(ApiError):
status = 404
code = "not_found"
default_message = "Объект не найден"
class Conflict(ApiError):
status = 409
code = "conflict"
default_message = "Конфликт"
class PayloadTooLarge(ApiError):
status = 413
code = "payload_too_large"
default_message = "Тело запроса слишком большое"
class UnsupportedMediaType(ApiError):
status = 415
code = "unsupported_media_type"
default_message = "Неподдерживаемый тип содержимого"
class Unavailable(ApiError):
status = 503
code = "unavailable"
default_message = "Сервис временно недоступен"
+19
View File
@@ -0,0 +1,19 @@
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`."""
from __future__ import annotations
import os
import sys
import urllib.request
def main() -> int:
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
return 0 if response.status == 200 else 1
except Exception:
return 1
if __name__ == "__main__":
sys.exit(main())
+328
View File
@@ -0,0 +1,328 @@
"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
Two stages, both with bounded memory and both BEFORE anything is stored:
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
(413) before the rest of the body is read and long before any pixel is decoded.
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
warning is fatal, so a truncated file or a file without scan data after SOS fails.
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
nothing here may turn user input into a 500.
"""
from __future__ import annotations
import asyncio
import os
import shutil
import struct
import zlib
from dataclasses import dataclass
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SOI = b"\xff\xd8\xff"
PNG = "image/png"
JPEG = "image/jpeg"
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
# Adam7 passes: (x start, y start, x step, y step).
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
# Standalone markers without a length field.
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
MAX_DIMENSION = 2**31 - 1
class ImageError(Exception):
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
def __init__(self, status: int, message: str):
super().__init__(message)
self.status = status
self.message = message
class VerifierUnavailable(Exception):
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
return ImageError(415, message)
@dataclass(frozen=True, slots=True)
class ImageInfo:
content_type: str
width: int
height: int
def detect(head: bytes) -> str | None:
"""Format by signature; None while fewer bytes than a signature have arrived."""
if head[:8] == PNG_SIGNATURE:
return PNG
if head[:3] == JPEG_SOI:
return JPEG
probe = min(len(head), 8)
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
return None
raise unsupported()
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
"""Format and size from the beginning of the file, or None if more bytes are needed.
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
kind = detect(head)
if kind is None:
return None
info = _png_header(head) if kind == PNG else _jpeg_header(head)
if info is None:
return None
if info.width * info.height > max_pixels:
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
return info
def _png_header(head: bytes) -> ImageInfo | None:
if len(head) < 33:
return None
length, ctype = struct.unpack(">I4s", head[8:16])
if ctype != b"IHDR" or length != 13:
raise unsupported()
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
raise unsupported()
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
raise unsupported()
return ImageInfo(PNG, width, height)
def _jpeg_header(head: bytes) -> ImageInfo | None:
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
i = 2
n = len(head)
while True:
if i >= n:
return None
if head[i] != 0xFF:
raise unsupported()
while i < n and head[i] == 0xFF: # fill bytes
i += 1
if i >= n:
return None
marker = head[i]
i += 1
if marker in JPEG_STANDALONE:
continue
if marker in (0xD8, 0xD9, 0xDA, 0x00):
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
raise unsupported()
if i + 2 > n:
return None
length = struct.unpack(">H", head[i:i + 2])[0]
if length < 2:
raise unsupported()
if marker in JPEG_SOF:
if i + 7 > n:
return None
if length < 8:
raise unsupported()
height, width = struct.unpack(">HH", head[i + 3:i + 7])
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
raise unsupported()
return ImageInfo(JPEG, width, height)
i += length
# ------------------------------------------------------------------ full verification
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
jpeg_max_memory_mb: int = 256) -> None:
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
if info.content_type == PNG:
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
else:
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
bpp = PNG_CHANNELS[color] * depth
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
segments = []
for xs, ys, dx, dy in passes:
pw = (width - xs + dx - 1) // dx if width > xs else 0
ph = (height - ys + dy - 1) // dy if height > ys else 0
if pw and ph:
stride = 1 + (pw * bpp + 7) // 8
segments.append((ph * stride, stride))
return segments
class _RawChecker:
"""Counts inflated bytes against the expected size and checks the filter byte of every row
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
VALID_FILTERS = bytes(range(5))
def __init__(self, segments: list[tuple[int, int]]):
self.segments = segments
self.index = 0 # current pass
self.offset = 0 # bytes already seen inside the current pass
self.expected = sum(size for size, _ in segments)
self.total = 0
def feed(self, piece: bytes) -> None:
self.total += len(piece)
if self.total > self.expected:
raise unsupported()
pos = 0
while pos < len(piece):
size, stride = self.segments[self.index]
take = min(size - self.offset, len(piece) - pos)
part = piece[pos:pos + take]
first = (-self.offset) % stride
if part[first::stride].translate(None, self.VALID_FILTERS):
raise unsupported()
self.offset += take
pos += take
if self.offset == size:
self.index += 1
self.offset = 0
def verify_png(data: bytes | bytearray | memoryview) -> None:
view = memoryview(data)
if bytes(view[:8]) != PNG_SIGNATURE:
raise unsupported()
pos = 8
n = len(view)
checker: _RawChecker | None = None
inflater = zlib.decompressobj()
color = None
seen_plte = seen_idat = idat_done = False
first = True
while True:
if pos + 12 > n:
raise unsupported() # truncated: no room for another chunk header + CRC
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
if length > n - pos - 12:
raise unsupported()
body = view[pos + 8:pos + 8 + length]
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
raise unsupported()
pos += 12 + length
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
raise unsupported()
if first:
if ctype != b"IHDR" or length != 13:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
raise unsupported()
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
first = False
continue
assert checker is not None
if ctype == b"IHDR":
raise unsupported()
if ctype == b"PLTE":
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise unsupported()
seen_plte = True
elif ctype == b"IDAT":
if idat_done:
raise unsupported() # IDAT chunks must be consecutive
if color == 3 and not seen_plte:
raise unsupported()
seen_idat = True
_inflate(inflater, body, checker)
elif ctype == b"IEND":
if length != 0 or not seen_idat:
raise unsupported()
if pos != n:
raise unsupported() # bytes after IEND (polyglot files) are not accepted
break
elif not 97 <= ctype[0] <= 122:
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
if ctype != b"IDAT" and seen_idat:
idat_done = True
# The zlib stream must end exactly with the last pixel row, without trailing data.
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
raise unsupported()
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
if inflater.eof:
if len(body):
raise unsupported() # data after the end of the zlib stream
return
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
try:
for start in range(0, len(body), INPUT_STEP):
pending: bytes | memoryview = body[start:start + INPUT_STEP]
while pending:
if inflater.eof:
raise unsupported() # data after the end of the zlib stream
checker.feed(inflater.decompress(pending, INFLATE_STEP))
pending = inflater.unconsumed_tail
if inflater.eof and inflater.unused_data:
raise unsupported()
except zlib.error:
raise unsupported() from None
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
buffers of progressive files); the separate process is killed at the deadline.
"""
process = await asyncio.create_subprocess_exec(
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
stderr = b""
try:
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
except (asyncio.TimeoutError, asyncio.CancelledError):
process.kill()
await process.wait()
raise
except (BrokenPipeError, ConnectionResetError):
await process.wait() # djpeg stopped reading: the file is already known to be broken
if process.returncode is not None and process.returncode < 0:
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
# SERVER, not of the file - never report it as 415.
raise VerifierUnavailable()
if process.returncode != 0:
if b"Insufficient memory" in (stderr or b""):
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
raise ImageError(413, "Изображение слишком велико для обработки")
raise unsupported()
+36
View File
@@ -0,0 +1,36 @@
"""JSON-lines logging to stdout. Logs never contain message texts or request bodies."""
from __future__ import annotations
import datetime as dt
import json
import logging
import sys
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
entry = {
"ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
"level": record.levelname.lower(),
"logger": record.name,
"event": record.getMessage(),
}
fields = getattr(record, "fields", None)
if fields:
entry.update(fields)
if record.exc_info:
# Stack traces go only to the server log, never to the HTTP client.
entry["exception"] = self.formatException(record.exc_info)
return json.dumps(entry, ensure_ascii=False)
def configure(level: str) -> None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
root.handlers[:] = [handler]
root.setLevel(level.upper())
# uvicorn's own access log is disabled; our middleware writes one line per request.
for name in ("uvicorn", "uvicorn.error"):
logging.getLogger(name).handlers[:] = []
logging.getLogger(name).propagate = True
+76
View File
@@ -0,0 +1,76 @@
"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`).
* Files `migrations/NNNN_name.sql` are applied in name order, each in its own
transaction together with its row in `schema_migrations` (DDL is transactional
in PostgreSQL, so a failed migration leaves nothing behind).
* Re-running is safe: applied versions are skipped, so data is never duplicated.
* The SHA-256 of every applied file is stored; if an applied file was edited later,
the run stops instead of silently diverging from the real schema.
* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once).
"""
from __future__ import annotations
import hashlib
import sys
import time
from pathlib import Path
import psycopg
from .config import ConfigError, _secret
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id
def say(text: str) -> None:
print(f"migrate: {text}", flush=True)
def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection:
deadline = time.monotonic() + wait_seconds
while True:
try:
return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate")
except psycopg.OperationalError as exc:
if time.monotonic() > deadline:
raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None
time.sleep(1)
def main() -> int:
try:
url = _secret("DATABASE_URL", required=True)
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
with connect(url) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY, checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())")
applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall())
for path in files:
version, sql = path.stem, path.read_bytes()
checksum = hashlib.sha256(sql).hexdigest()
if version in applied:
if applied[version] != checksum:
say(f"{version}: file changed after it was applied; create a new migration instead")
return 1
say(f"{version}: already applied, skipped")
continue
with conn.transaction():
conn.execute(sql.decode("utf-8"))
conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum))
say(f"{version}: applied")
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
say(f"done, {len(files)} migration file(s) checked")
return 0
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7).
* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt
there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole.
* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker
thread (argon2-cffi releases the GIL), so the event loop keeps serving other
requests, and a semaphore caps parallel hashes (memory/CPU DoS protection).
* An unknown username is checked against a dummy hash, so "no such user" and
"wrong password" take the same time and return the same 401.
"""
from __future__ import annotations
import asyncio
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16)
_semaphore: asyncio.Semaphore | None = None
_dummy_hash: str | None = None
def configure(concurrency: int) -> None:
global _semaphore
_semaphore = asyncio.Semaphore(concurrency)
async def _run(func, *args):
if _semaphore is None:
configure(2)
async with _semaphore:
return await asyncio.to_thread(func, *args)
async def hash_password(password: str) -> str:
return await _run(_hasher.hash, password)
def _verify(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored_hash: str | None, password: str) -> bool:
global _dummy_hash
if stored_hash is None:
if _dummy_hash is None:
_dummy_hash = await hash_password("dummy-password-for-timing")
await _run(_verify, _dummy_hash, password)
return False
return await _run(_verify, stored_hash, password)
+437
View File
@@ -0,0 +1,437 @@
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
docker compose run --rm reconciler # dry-run: report only (default)
docker compose run --rm reconciler --apply # fix what is safe to fix
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
What it finds and what --apply does (nothing else is ever changed):
kind condition --apply
orphan_object object, no row, older than --grace delete the object
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
missing_object row 'ready', object absent, older than --grace quarantine the row
size_mismatch row 'ready', object size != size_bytes quarantine the row
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
restored_object row quarantined, object present and correct again remove the quarantine
(a checksum quarantine only with --verify-checksums)
fresh_* the same, but younger than --grace nothing (in-flight upload)
Safety:
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
the planner cannot modify a store;
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
which is reversible; objects that have a row are never deleted unless the row is a stale
intent; anything younger than --grace is never touched;
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
where a row exists, so the live API racing with the reconciler cannot be hurt;
* object first, row second: an interrupted run leaves a state the next run repairs;
re-running is idempotent (the second run finds nothing to do);
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
"""
from __future__ import annotations
import argparse
import asyncio
import datetime as dt
import hashlib
import json
import sys
import time
from collections import Counter
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
import psycopg
from .config import ConfigError, Settings, _float, _secret, s3_settings
from .errors import Unavailable
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
def log(text: str) -> None:
print(f"reconcile: {text}", file=sys.stderr, flush=True)
@dataclass(frozen=True, slots=True)
class Row:
id: str
key: str
status: str
size: int
sha256: bytes
created_at: dt.datetime
updated_at: dt.datetime
lost_reason: str | None
used: bool
@dataclass(slots=True)
class Finding:
kind: str
key: str
attachment_id: str | None = None
used: bool = False
detail: str = ""
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
@dataclass
class Stats:
objects: int = 0
rows: int = 0
consistent: int = 0
s3_list_requests: int = 0
s3_head_requests: int = 0
s3_get_bytes: int = 0
db_pages: int = 0
class ReadOnlyStorage:
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
def __init__(self, storage: Storage):
self._s = storage
async def list_page(self, *args, **kwargs):
return await self._s.list_page(*args, **kwargs)
async def head(self, key: str):
return await self._s.head(key)
async def read_all(self, key: str, limit: int) -> bytes:
return await self._s.read_all(key, limit)
class Reconciler:
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
page_size: int, verify: bool, apply: bool):
self.conn = conn
self.s3 = storage
self.grace = grace
self.page_size = page_size
self.verify = verify
self.apply = apply
self.stats = Stats()
self.findings: list[Finding] = []
# ---------------------------------------------------------- streams
async def objects(self) -> AsyncIterator[ListedObject]:
after = None
while True:
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
self.stats.s3_list_requests += 1
for item in page:
self.stats.objects += 1
yield item
if not more or not page:
return
after = page[-1].key
async def rows(self) -> AsyncIterator[Row]:
after = ""
while True:
cur = await self.conn.execute(
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
(after, self.page_size))
page = await cur.fetchall()
self.stats.db_pages += 1
for r in page:
self.stats.rows += 1
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
if len(page) < self.page_size:
return
after = page[-1][1]
# ---------------------------------------------------------- planning (read-only)
async def scan(self) -> None:
cur = await self.conn.execute("SELECT clock_timestamp()")
self.db_now = (await cur.fetchone())[0]
self.s3_now = time.time()
objects, rows = self.objects(), self.rows()
obj = await anext(objects, None)
row = await anext(rows, None)
while obj is not None or row is not None:
if row is None or (obj is not None and obj.key < row.key):
self.object_only(obj)
obj = await anext(objects, None)
elif obj is None or row.key < obj.key:
self.row_only(row)
row = await anext(rows, None)
else:
await self.both(obj, row)
obj = await anext(objects, None)
row = await anext(rows, None)
def row_old(self, row: Row) -> bool:
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
def object_only(self, obj: ListedObject) -> None:
old = self.s3_now - obj.last_modified_ts > self.grace
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
def row_only(self, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object absent"))
elif row.lost_reason is not None:
self.stats.consistent += 1 # already quarantined, still missing: nothing new
else:
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
used=row.used))
async def both(self, obj: ListedObject, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object present"))
return
problem = await self.object_problem(row, obj.size)
if problem:
if row.lost_reason is None:
self.add(Finding(problem, row.key, row.id, used=row.used))
else:
self.stats.consistent += 1 # quarantined and still broken
elif row.lost_reason == "checksum_mismatch" and not self.verify:
# Same size, but this pass does not read the bytes: it cannot prove the object is the
# original again, so a checksum quarantine is lifted only by --verify-checksums.
self.stats.consistent += 1
elif row.lost_reason is not None:
self.add(Finding("restored_object", row.key, row.id, used=row.used))
else:
self.stats.consistent += 1
async def object_problem(self, row: Row, size: int) -> str | None:
if size != row.size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(row.key, row.size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != row.sha256:
return "checksum_mismatch"
return None
def add(self, finding: Finding) -> None:
if finding.kind not in ACTION_OF:
finding.result = "not_actionable"
self.findings.append(finding)
# ---------------------------------------------------------- actions (--apply only)
async def run_actions(self, max_orphan_fraction: float) -> None:
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
for finding in self.findings:
action = ACTION_OF.get(finding.kind)
if action is None:
continue
try:
finding.result = await handlers[action](finding)
except (Unavailable, psycopg.OperationalError) as exc:
finding.result = f"failed:{type(exc).__name__}"
raise # a store is down: stop; the next run continues from the committed state
async def delete_object(self, f: Finding) -> str:
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
if await cur.fetchone():
return "skipped:row_appeared"
try:
head = await self.s3.head(f.key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "skipped:already_gone"
if time.time() - head.last_modified_ts <= self.grace:
return "skipped:object_rewritten_recently"
# No row names this key and the object is older than the grace period. Rows are always
# written BEFORE their object and keys are random and never reused, so no upload can
# still claim it.
await self.s3.delete(f.key)
return "applied"
async def delete_intent(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
(f.attachment_id, self.grace))
if await cur.fetchone() is None:
return "skipped:no_longer_stale_intent"
await self.s3.delete(f.key) # object first (idempotent), then the row
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
(f.attachment_id,))
return "applied"
async def quarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
if reason is None:
return "skipped:object_ok_now"
await self.conn.execute(
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
(reason, f.attachment_id))
return "applied"
async def unquarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
(f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
if row[2] == "checksum_mismatch" and not self.verify:
return "skipped:needs_verify_checksums"
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
return "skipped:object_still_bad"
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
(f.attachment_id,))
return "applied"
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
"""Fresh look at the object right before changing the row (the listing may be stale)."""
try:
head = await self.s3.head(key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "missing_object"
if head.size != size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(key, size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != sha256:
return "checksum_mismatch"
return None
# ---------------------------------------------------------- report
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
kinds = Counter(f.kind for f in self.findings)
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
samples = {}
for f in self.findings:
items = samples.setdefault(f.kind, [])
if len(items) < sample:
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
"result": f.result, **({"detail": f.detail} if f.detail else {})})
return {
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
"duration_seconds": round(seconds, 3), "error": error,
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
"findings": {k: kinds.get(k, 0) for k in KINDS},
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
"failed": results.get("failed", 0)},
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
"page_size": self.page_size},
"samples": samples,
}
class SafetyValve(Exception):
pass
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
p.add_argument("--grace", type=float, default=3600.0,
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
p.add_argument("--allow-small-grace", action="store_true",
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
"intent of an upload that is still running; that upload then fails with 503)")
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
args = p.parse_args(argv)
if not 1 <= args.page_size <= 1000:
p.error("--page-size must be 1..1000")
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
if not 0 < args.max_orphan_fraction <= 1:
p.error("--max-orphan-fraction must be in (0, 1]")
return args
async def run(args: argparse.Namespace) -> int:
url = _secret("DATABASE_URL", required=True)
base = s3_settings()
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
database_url=url, **base)
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
if not args.apply:
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
mode = "apply" if args.apply else "dry-run"
storage = Storage(settings)
await storage.open()
started = time.perf_counter()
error = None
code = 0
try:
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
application_name=f"messenger-reconcile-{mode}")
except psycopg.OperationalError as exc:
await storage.close()
error = f"store_unavailable: database {type(exc).__name__}"
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
log(error)
return 1
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
try:
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
await rec.scan()
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
if args.apply:
await rec.run_actions(args.max_orphan_fraction)
except SafetyValve as exc:
error, code = f"safety_valve: {exc}", 3
except (Unavailable, psycopg.OperationalError) as exc:
error, code = f"store_unavailable: {type(exc).__name__}", 1
finally:
await conn.close()
await storage.close()
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
indent=1))
if error:
log(error)
return code
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+372
View File
@@ -0,0 +1,372 @@
"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
values travel separately from the SQL text, so input can never change the query.
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
"""
from __future__ import annotations
import datetime as dt
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from .cursor import SortKey
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
from .timefmt import from_us, rfc3339, to_us
def _b(text: str) -> bytes:
return text.encode("utf-8")
def _s(raw: bytes | memoryview) -> str:
return bytes(raw).decode("utf-8")
@dataclass(slots=True, frozen=True)
class User:
id: str
username: str
display_name: str
created_at: dt.datetime
def to_json(self) -> dict:
return {"id": self.id, "username": self.username, "display_name": self.display_name,
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: dt.datetime
attachment_ids: tuple[str, ...] = ()
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
"created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)}
def _user(row) -> User:
return User(str(row[0]), row[1], _s(row[2]), row[3])
# ------------------------------------------------------------------ users
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
async def work(conn: psycopg.AsyncConnection) -> User:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
"RETURNING id, username, display_name, created_at",
(uuid.uuid4(), username, _b(display_name), password_hash))
return _user(await cur.fetchone())
try:
return await db.run(work)
except pg_errors.UniqueViolation:
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
raise Conflict("Имя пользователя уже занято") from None
async def get_user(db: Database, user_id: str) -> User | None:
async def work(conn: psycopg.AsyncConnection) -> User | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
row = await cur.fetchone()
return _user(row) if row else None
return await db.run(work)
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,))
row = await cur.fetchone()
return (_user(row), row[4]) if row else None
return await db.run(work)
# ------------------------------------------------------------------ chats
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
"""Atomic: the chat row and all memberships commit together or not at all.
The chat is inserted FIRST; an unknown member then fails the membership INSERT
with a foreign-key violation and the whole transaction (chat row included) is
rolled back. student-tests check that nothing is left behind.
"""
if creator_id not in member_ids:
raise InvalidRequest("member_ids должен включать текущего пользователя")
chat_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
(chat_id, _b(title), creator_id))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
"SELECT %s, m.user_id, m.ord - 1, %s "
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
(chat_id, created_at, member_ids))
return created_at
try:
created_at = await db.run(work)
except pg_errors.ForeignKeyViolation:
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
return Chat(str(chat_id), title, tuple(member_ids), created_at)
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
if not rows:
return []
ids = [r[0] for r in rows]
cur = await conn.execute(
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
for chat_id, user_id in await cur.fetchall():
members[chat_id].append(str(user_id))
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
async def work(conn: psycopg.AsyncConnection) -> Chat:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Чат не найден")
return (await _load_chats(conn, [row]))[0]
return await db.run(work)
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
if await cur.fetchone() is None:
raise NotFound("Чат не найден")
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await _check_member(conn, user_id, chat_id)
await db.run(work)
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
if after is None:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, limit + 1))
else:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, from_us(after[0]), after[1], limit + 1))
rows = await cur.fetchall()
return await _load_chats(conn, rows[:limit]), len(rows) > limit
return await db.run(work)
# ------------------------------------------------------------------ messages
@dataclass(slots=True, frozen=True)
class PostResult:
message: Message
replayed: bool # True: the key was already used with this body, nothing new was written
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
text: str, attachment_ids: list[str]) -> dt.datetime:
"""Insert with a per-chat strictly increasing timestamp.
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
chat are serialised: a message with a smaller created_at can never commit after
a message with a bigger one, and a reader paging by cursor cannot skip it.
"""
cur = await conn.execute(
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
(chat_id,))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) "
"VALUES (%s, %s, %s, %s, %s, %s::uuid[])",
(message_id, chat_id, user_id, _b(text), created_at, attachment_ids))
if attachment_ids:
await conn.execute(
"INSERT INTO message_attachments (message_id, position, attachment_id) "
"SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)",
(message_id, attachment_ids))
return created_at
# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409).
# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT.
CHECK_ATTACHMENTS = (
"SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM ("
"SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' "
"FOR SHARE) a")
async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str:
if not attachment_ids:
return "ok"
cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id))
found, usable = await cur.fetchone()
if found != len(attachment_ids):
return "attachment_not_found"
return "ok" if usable == found else "attachment_conflict"
def _attachment_error(status: str) -> None:
if status == "attachment_not_found":
raise NotFound("Вложение не найдено")
if status == "attachment_conflict":
raise Conflict("Вложение недоступно и не может быть прикреплено")
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
CLAIM_KEY = (
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
"VALUES (%s, %s, %s, %s, %s) "
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
"RETURNING 1")
STORED_RESULT = (
"SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k "
"JOIN messages m ON m.id = k.message_id "
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
ever left behind: a crash or 5xx rolls everything back and the client may retry);
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
* same key, other body -> 409; the membership check comes first, so outsiders always get 404;
* attachments (lab 4) are checked before the key is claimed (same order as post_message_v2).
"""
message_id = uuid.uuid4()
attachment_ids = list(attachment_ids)
async def work(conn: psycopg.AsyncConnection) -> PostResult:
await _check_member(conn, user_id, chat_id)
check = await _check_attachments(conn, chat_id, attachment_ids)
if check == "ok":
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
if await cur.fetchone() is not None:
created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids)
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at,
tuple(attachment_ids)), False)
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
row = await cur.fetchone()
if row is None:
_attachment_error(check)
# The key row vanished between the two statements (expired-key cleanup): retryable.
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
if bytes(row[0]) != request_hash:
raise Conflict("Idempotency-Key уже использован с другим сообщением")
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3],
tuple(str(a) for a in row[4])), True)
return await db.run(work)
POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments "
"FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])")
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
"""Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 =
0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction."""
message_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> tuple:
cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text),
ttl_seconds, list(attachment_ids)))
return await cur.fetchone()
status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False)
if status == "not_found":
raise NotFound("Чат не найден")
_attachment_error(status)
if status == "conflict":
raise Conflict("Idempotency-Key уже использован с другим сообщением")
if status == "retry":
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at,
tuple(str(a) for a in stored_attachments)), status == "replayed")
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
"SELECT ctid FROM idempotency_keys "
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
(ttl_seconds, batch))
return cur.rowcount
return await db.run(work)
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
# would push the cursor into a Filter and late pages would scan the whole prefix.
# attachment_ids is a column of the message row (migration 0004): still one index range scan.
HISTORY_FIRST = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
HISTORY_AFTER = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
async def work(conn: psycopg.AsyncConnection) -> list:
if after is None:
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
else:
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
return await cur.fetchall()
rows = await db.run(work)
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4]))
for r in rows[:limit]]
return items, len(rows) > limit
+151
View File
@@ -0,0 +1,151 @@
"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`).
Idempotent, safe to run on every `docker compose up`:
1. waits until the S3 endpoint answers;
2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and
its objects are left untouched);
3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs;
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
* anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused;
* the API key pair can read/write objects of ITS bucket, but can neither list it, nor create
a bucket, nor see or read another bucket (a canary bucket created here for this check);
* the reconciler key pair can list the bucket but cannot create one.
Only file names and verdicts are printed, never key values.
"""
from __future__ import annotations
import asyncio
import os
import sys
import time
import urllib.error
import urllib.request
from botocore.exceptions import ClientError
from .config import ConfigError, Settings, _secret, s3_settings
from .errors import Unavailable
from .storage import SENTINEL_KEY, Storage
def say(text: str) -> None:
print(f"s3-bootstrap: {text}", flush=True)
def anonymous_status(url: str) -> int:
"""HTTP status of an UNSIGNED request (what anyone on the network could do)."""
try:
with urllib.request.urlopen(url, timeout=3) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
async def error_code(coro) -> str:
"""S3 error code of a call that is expected to be refused ('' if it succeeded)."""
try:
await coro
return ""
except ClientError as exc:
return str(exc.response.get("Error", {}).get("Code", "error"))
def settings_with(keys: tuple[str, str]) -> Settings:
base = s3_settings(credentials_required=False)
base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1])
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
async def run() -> int:
admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True))
api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True))
reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True))
settings = settings_with(admin)
bucket = settings.s3_bucket
admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler))
for s in (admin_s3, api_s3, rec_s3):
await s.open()
try:
deadline = time.monotonic() + 60
while True:
try:
await admin_s3._call("list_buckets", admin_s3.client.list_buckets())
break
except Unavailable:
if time.monotonic() > deadline:
say("S3 endpoint does not answer within 60 s")
return 1
await asyncio.sleep(1)
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
if code == "":
say(f"bucket {bucket}: exists, kept as is")
elif code in ("404", "NoSuchBucket", "NotFound"):
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
say(f"bucket {bucket}: create failed ({code})")
return 1
say(f"bucket {bucket}: created")
else:
say(f"bucket {bucket}: head failed ({code})")
return 1
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
say(f"sentinel {SENTINEL_KEY}: written")
canary = bucket + "-canary"
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
await admin_s3.client.create_bucket(Bucket=canary)
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
failures = []
base = settings.s3_endpoint.rstrip("/")
for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"),
("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")):
status = anonymous_status(url)
verdict = "refused" if status in (401, 403) else "ALLOWED"
say(f"{what}: HTTP {status} -> {verdict}")
if verdict != "refused":
failures.append(what)
probe = "health/bootstrap-probe"
checks = [
("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True),
("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True),
("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True),
("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False),
("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False),
("api: read object of another bucket",
await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False),
("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True),
("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False),
]
for what, code, allowed in checks:
ok = (code == "") == allowed
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
if not ok:
failures.append(what)
visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])]
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
if canary in visible:
failures.append("api sees another bucket")
if failures:
say("access model is WRONG: " + ", ".join(failures))
return 1
say("done: private bucket, least-privilege keys verified")
return 0
finally:
for s in (admin_s3, api_s3, rec_s3):
await s.close()
def main() -> int:
try:
return asyncio.run(run())
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
os.environ.setdefault("PYTHONUNBUFFERED", "1")
sys.exit(main())
+160
View File
@@ -0,0 +1,160 @@
"""Server-side sessions behind an opaque Bearer token.
* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is
stored, so a leaked database/AOF does not contain usable tokens. A fast hash is
enough for a 256-bit random value; passwords need a slow KDF (passwords.py).
* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup
compares it with the current time, and a restart never extends it.
Two interchangeable stores (SESSION_STORE):
* `valkey` (compose default): the ONLY copy of a session is a Valkey key with
`PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second
copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked
token cannot come back after Valkey fails and recovers. If Valkey is unreachable
a session cannot be checked: protected requests and logout get 503, never access.
* `postgres`: table `sessions` (migrations/0001_init.sql).
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import json
import logging
import secrets
from typing import Protocol
from redis import exceptions as redis_errors
from redis.asyncio import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from .config import Settings
from .db import Database
from .errors import Unavailable
from .timefmt import UTC, from_us, to_us
log = logging.getLogger("messenger.sessions")
def new_token() -> str:
return secrets.token_urlsafe(32)
def token_hash(token: str) -> bytes:
return hashlib.sha256(token.encode("ascii")).digest()
def now() -> dt.datetime:
return dt.datetime.now(UTC)
class SessionStore(Protocol):
name: str
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ...
async def user_id_for(self, token: str) -> str | None: ...
async def revoke(self, token: str) -> bool: ...
async def ping(self) -> bool: ...
async def close(self) -> None: ...
class PostgresSessionStore:
name = "postgres"
def __init__(self, db: Database):
self.db = db
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
async def work(conn) -> None:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash(token), user_id, expires_at))
# Housekeeping: drop this user's expired sessions (bounded by the user's own rows).
await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now()))
await self.db.run(work)
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
async def work(conn):
cur = await conn.execute(
"SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return await cur.fetchone()
row = await self.db.run(work)
return str(row[0]) if row else None
async def revoke(self, token: str) -> bool:
async def work(conn) -> bool:
cur = await conn.execute(
"DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return cur.rowcount == 1
return await self.db.run(work)
async def ping(self) -> bool:
return True # covered by the database readiness check
async def close(self) -> None:
return None
class ValkeySessionStore:
name = "valkey"
PREFIX = "session:"
def __init__(self, settings: Settings):
self.timeout = settings.cache_timeout_seconds
self.client = Redis.from_url(
settings.cache_url,
socket_timeout=self.timeout,
socket_connect_timeout=self.timeout,
retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once
decode_responses=False,
)
def _key(self, token: str) -> str:
return self.PREFIX + token_hash(token).hex()
async def _call(self, coro):
try:
async with asyncio.timeout(self.timeout * 2):
return await coro
except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc:
log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("Хранилище сессий временно недоступно") from None
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode()
# PXAT = absolute expiry time: survives restarts (AOF) without being extended.
await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000))
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
raw = await self._call(self.client.get(self._key(token)))
if raw is None:
return None
data = json.loads(raw)
if from_us(data["expires_at_us"]) <= now():
return None # do not rely only on Valkey's lazy expiry
return data["user_id"]
async def revoke(self, token: str) -> bool:
return await self._call(self.client.delete(self._key(token))) == 1
async def ping(self) -> bool:
try:
return bool(await self._call(self.client.ping()))
except Unavailable:
return False
async def close(self) -> None:
await self.client.aclose()
def build_store(settings: Settings, db: Database) -> SessionStore:
return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db)
+177
View File
@@ -0,0 +1,177 @@
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3
restarts, pooled keep-alive connections to the old process are dead and the first request on
such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean
clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen
key, GET, HEAD, DELETE), so repeating it cannot create a second attachment.
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
plain, widely supported S3 requests.
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import hashlib
import logging
from collections.abc import AsyncIterator
from dataclasses import dataclass
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.storage")
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
class ObjectMissing(Exception):
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
@dataclass(frozen=True, slots=True)
class ObjectHead:
size: int
content_type: str
metadata: dict[str, str]
last_modified_ts: float # POSIX seconds
@dataclass(frozen=True, slots=True)
class ListedObject:
key: str
size: int
last_modified_ts: float # POSIX seconds
class Storage:
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
self.bucket = settings.s3_bucket
self.timeout = settings.s3_timeout_seconds
self._endpoint = settings.s3_endpoint
self._region = settings.s3_region
self._access_key = access_key or settings.s3_access_key_id
self._secret_key = secret_key or settings.s3_secret_access_key
self._config = AioConfig(
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_timeout_seconds,
retries={"total_max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
signature_version="s3v4",
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
max_pool_connections=settings.s3_max_connections,
)
self._stack: contextlib.AsyncExitStack | None = None
self._client = None
async def open(self) -> None:
self._stack = contextlib.AsyncExitStack()
session = get_session()
self._client = await self._stack.enter_async_context(session.create_client(
"s3", endpoint_url=self._endpoint, region_name=self._region,
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
async def close(self) -> None:
if self._stack is not None:
await self._stack.aclose()
self._stack = None
@property
def client(self):
assert self._client is not None, "Storage.open() was not called"
return self._client
async def _call(self, what: str, coro, deadline: float | None = None):
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
try:
return await asyncio.wait_for(coro, deadline or self.timeout)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code in MISSING_CODES:
raise ObjectMissing() from None
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
raise Unavailable("Хранилище файлов временно недоступно") from None
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
raise Unavailable("Хранилище файлов временно недоступно") from None
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
await self._call("put", self.client.put_object(
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
ContentMD5=md5, Metadata=metadata), deadline)
async def head(self, key: str) -> ObjectHead:
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
download instead of hanging the connection)."""
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
body = response["Body"]
size = int(response["ContentLength"])
async def chunks() -> AsyncIterator[bytes]:
try:
while True:
chunk = await self._call("get_body", body.read(64 * 1024))
if not chunk:
return
yield chunk
finally:
body.close()
return size, chunks()
async def read_all(self, key: str, limit: int) -> bytes:
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
size, chunks = await self.open_stream(key)
if size > limit:
raise Unavailable("Объект больше допустимого размера")
parts = [chunk async for chunk in chunks]
return b"".join(parts)
async def delete(self, key: str) -> None:
"""Idempotent: deleting a missing key is a success in S3."""
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
if start_after:
params["StartAfter"] = start_after
response = await self._call("list", self.client.list_objects_v2(**params))
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
for o in response.get("Contents", [])]
return items, bool(response.get("IsTruncated"))
async def ping(self, timeout: float = 1.0) -> bool:
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
answers, the bucket exists and the API credentials are accepted (Read permission)."""
try:
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
return True
except (Unavailable, ObjectMissing):
return False
SENTINEL_KEY = "health/sentinel"
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>
+22
View File
@@ -0,0 +1,22 @@
"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings."""
from __future__ import annotations
import datetime as dt
UTC = dt.timezone.utc
_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC)
def to_us(value: dt.datetime) -> int:
"""Microseconds since the Unix epoch (exact: timestamptz has microsecond precision)."""
delta = value - _EPOCH
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
def from_us(us: int) -> dt.datetime:
return _EPOCH + dt.timedelta(microseconds=us)
def rfc3339(value: dt.datetime) -> str:
"""RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z."""
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+199
View File
@@ -0,0 +1,199 @@
"""Strict request validation according to contracts/openapi.json + contracts/README.md.
Written by hand instead of a schema library so that every rule is visible:
extra fields, wrong JSON types, whitespace-only strings and lengths counted in
Unicode code points all produce 400 invalid_request.
"""
from __future__ import annotations
import json
import re
import unicodedata
from typing import Any
from urllib.parse import parse_qsl
from .errors import InvalidRequest
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case.
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
LIMIT_RE = re.compile(r"[0-9]{1,3}")
# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}.
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
SURROGATES_RE = re.compile("[\ud800-\udfff]")
DEFAULT_LIMIT = 50
MAX_LIMIT = 100
def _reject_constant(name: str) -> Any:
# Python's json accepts NaN/Infinity, which are not valid JSON.
raise ValueError(f"invalid JSON constant {name}")
def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
# {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins".
result = dict(pairs)
if len(result) != len(pairs):
raise ValueError("duplicate key in JSON object")
return result
def parse_json_object(body: bytes) -> dict[str, Any]:
"""Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400)."""
try:
text = body.decode("utf-8")
value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys)
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None
if not isinstance(value, dict):
raise InvalidRequest("Тело запроса должно быть JSON-объектом")
return value
def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None:
extra = set(data) - required - optional
if extra:
raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra)))
missing = required - set(data)
if missing:
raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing)))
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str:
value = data[name]
if not isinstance(value, str):
raise InvalidRequest(f"Поле {name} должно быть строкой")
if SURROGATES_RE.search(value):
# "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8.
raise InvalidRequest(f"Поле {name} содержит недопустимые символы")
# len() of a Python str is the number of Unicode code points, as the contract requires.
if not min_len <= len(value) <= max_len:
raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов")
if value.strip() == "" and not allow_blank:
raise InvalidRequest(f"Поле {name} не может состоять только из пробелов")
return value
def username_field(data: dict[str, Any]) -> str:
value = data["username"]
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}")
return value
def password_field(data: dict[str, Any]) -> str:
"""12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed."""
value = data["password"]
if not isinstance(value, str):
raise InvalidRequest("Поле password должно быть строкой")
if SURROGATES_RE.search(value):
raise InvalidRequest("Поле password содержит недопустимые символы")
if not 12 <= len(value) <= 128:
raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов")
if value.strip() == "":
raise InvalidRequest("Пароль не может состоять только из пробелов")
return value
def parse_uuid(value: Any, what: str = "UUID") -> str:
if not isinstance(value, str) or UUID_RE.fullmatch(value) is None:
raise InvalidRequest(f"Неверный {what}")
return value.lower()
def member_ids_field(data: dict[str, Any]) -> list[str]:
value = data["member_ids"]
if not isinstance(value, list):
raise InvalidRequest("member_ids должен быть массивом")
if not 1 <= len(value) <= 100:
raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(item, "UUID участника") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("member_ids содержит дубликаты")
return ids
def attachment_ids_field(data: dict[str, Any]) -> list[str]:
"""Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md);
null or any other non-array type is 400."""
value = data.get("attachment_ids", [])
if not isinstance(value, list):
raise InvalidRequest("attachment_ids должен быть массивом")
if len(value) > 10:
raise InvalidRequest("attachment_ids содержит больше 10 элементов")
ids = [parse_uuid(item, "UUID вложения") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("attachment_ids содержит дубликаты")
return ids
def idempotency_key(raw: str | None) -> str:
if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None:
raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}")
return raw
def single_query_param(query: list[tuple[str, str]], name: str) -> str | None:
values = [v for k, v in query if k == name]
if len(values) > 1:
raise InvalidRequest(f"Параметр {name} указан несколько раз")
return values[0] if values else None
def parse_limit(raw: str | None) -> int:
if raw is None:
return DEFAULT_LIMIT
if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}")
return int(raw)
# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown
# to other users, so characters that change how a name LOOKS are refused: control characters
# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls
# ("photo<U+202E>gnp.exe" would display as "photoexe.png").
BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"})
def query_items(raw: bytes) -> list[tuple[str, str]]:
"""Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD)."""
try:
return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict")
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None
def filename_param(query: list[tuple[str, str]]) -> str:
value = single_query_param(query, "filename")
if value is None:
raise InvalidRequest("Параметр filename обязателен")
if not 1 <= len(value) <= 255:
raise InvalidRequest("Длина filename должна быть от 1 до 255 символов")
if value.strip() == "" or value in {".", ".."}:
raise InvalidRequest("filename не может состоять только из пробелов или точек")
if "/" in value or "\\" in value:
raise InvalidRequest("filename не может содержать / или \\")
for char in value:
if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS:
raise InvalidRequest("filename содержит управляющие символы")
return value
ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4
def variant_param(query: list[tuple[str, str]]) -> str:
"""`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant
name is a missing variant of a ready object (404); anything else is an invalid value (400)."""
value = single_query_param(query, "variant")
if value is None or value == "original":
return "original"
if value in ATTACHMENT_VARIANTS:
raise NotFoundVariant()
raise InvalidRequest("variant должен быть original")
class NotFoundVariant(Exception):
"""Raised for a variant that lab 4 does not produce; mapped to 404 by the handler."""
+380
View File
@@ -0,0 +1,380 @@
"""Shared helpers for the student tests.
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
`docker compose` for database checks (psql inside the postgres container), the HAProxy
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
python -m pytest student-tests # everything (stand must be up)
python -m pytest student-tests -m "not stack" # unit only
"""
from __future__ import annotations
import base64
import contextlib
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import quote
import httpx2 as httpx
import pytest
from jsonschema import Draft202012Validator
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "scripts"))
import lbctl # noqa: E402
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
def pytest_configure(config):
config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)")
def schema_errors(value, name: str) -> list[str]:
"""Validate against components/schemas/<name> of the course contract (JSON Schema 2020-12)."""
schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]}
validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
return [e.message for e in validator.iter_errors(value)]
def basic(username: str, password: str) -> str:
return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
# ------------------------------------------------------------------ docker / psql
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", check=check, timeout=timeout, input=stdin)
def psql(sql: str) -> str:
"""Run SQL inside the postgres container (no DB port is published) and return unaligned output."""
result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-v", "ON_ERROR_STOP=1", "-c", sql)
return result.stdout.strip()
def wait_ready(timeout: float = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.3)
raise AssertionError("stand did not become ready")
def wait_healthy(*services: str, timeout: float = 120) -> None:
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
deadline = time.monotonic() + timeout
pending = set(services)
while pending and time.monotonic() < deadline:
for service in list(pending):
ids = compose("ps", "-q", service, check=False).stdout.split()
if ids:
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
capture_output=True, text=True).stdout.split()
if states and all(state == "healthy" for state in states):
pending.discard(service)
time.sleep(0.5)
assert not pending, f"not healthy: {pending}"
def wait_stand(replicas: int = REPLICAS) -> None:
"""Readiness through the LB AND every replica back in HAProxy rotation."""
wait_ready()
lbctl.wait_up(replicas, timeout=90)
def restart_services(*services: str) -> None:
compose("restart", *services)
wait_healthy(*services)
wait_stand()
@contextlib.contextmanager
def service_down(service: str, how: str = "stop"):
"""Stop or pause a compose service; ALWAYS bring it back and wait for readiness."""
assert how in {"stop", "pause"}
compose(how, service)
try:
yield
finally:
compose("unpause" if how == "pause" else "start", service)
wait_healthy(service)
wait_stand()
@contextlib.contextmanager
def api_with_env(**overrides: str):
"""Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2),
then ALWAYS recreate it with the normal configuration."""
env = {**os.environ, **overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
capture_output=True, text=True, check=True, timeout=180)
wait_stand()
try:
yield
finally:
clean = {k: v for k, v in os.environ.items() if k not in overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
capture_output=True, text=True, check=True, timeout=180)
wait_stand()
def running_replicas() -> list[dict]:
return [r for r in lbctl.replicas() if r["state"] == "running"]
@contextlib.contextmanager
def replica_stopped(replica: dict, mode: str = "stop"):
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
and wait until HAProxy has every replica UP."""
assert mode in {"stop", "kill"}
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
capture_output=True, text=True, check=True, timeout=60)
try:
yield
finally:
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
wait_healthy("api")
wait_stand()
def api_env(name: str) -> str:
"""Current value of an environment variable inside the running api container."""
return compose("exec", "-T", "api", "printenv", name).stdout.strip()
# ------------------------------------------------------------------ HTTP
class Api:
"""HTTP helper that checks the invariants every response must satisfy."""
def __init__(self, base_url: str = BASE_URL):
self.client = httpx.Client(base_url=base_url, timeout=15)
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
lb_error_ok: bool = False):
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
accepted without X-Instance-Id; every response of an API replica must carry it."""
h = dict(headers or {})
if token is not None:
h["Authorization"] = "Bearer " + token
if json_body is not None:
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
response = self.client.request(method, path, content=raw, headers=h)
allowed = (expected,) if isinstance(expected, int) else expected
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
request_id = response.headers.get("x-request-id")
assert request_id, "every response must carry X-Request-Id"
if not response.headers.get("x-instance-id"):
assert lb_error_ok and response.status_code in (502, 503, 504), \
("missing X-Instance-Id", method, path, response.status_code)
if response.status_code >= 400:
body = response.json()
assert schema_errors(body, "Error") == [], response.text
assert body["error"]["request_id"] == request_id
assert "Traceback" not in response.text and "psycopg" not in response.text
if response.status_code == 401:
challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer"
assert response.headers.get("www-authenticate", "").lower().startswith(challenge)
elif schema:
assert schema_errors(response.json(), schema) == [], response.text
return response
def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"[:32]
body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password}
user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json()
return {"id": user["id"], "user": user, "username": username, "password": password}
def login(self, actor: dict) -> str:
session = self.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])},
schema="Session").json()
assert session["user"] == actor["user"] and session["token_type"] == "Bearer"
return session["token"]
def user(self, label: str = "u", **kwargs) -> dict:
actor = self.register(label, **kwargs)
actor["token"] = self.login(actor)
return actor
def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict:
ids = [owner["id"], *(m["id"] for m in members)]
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
raw: bytes | None = None, headers: dict | None = None, **kwargs):
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
if raw is not None:
h.setdefault("Content-Type", "application/json")
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
json_body=body if raw is None else None, raw=raw, headers=h,
schema="Message" if expected == 201 else None, **kwargs)
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
return self.post_message(actor, chat, {"text": text}, key=key).json()
def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png",
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
h = dict(headers or {})
if content_type is not None:
h["Content-Type"] = content_type
if query is None:
query = "" if filename is None else "filename=" + quote(filename, safe="")
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
if isinstance(data, (bytes, bytearray)):
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
schema="Attachment" if expected == 201 else None)
h["Authorization"] = "Bearer " + actor["token"]
response = self.client.post(path, content=data, headers=h)
assert response.status_code == expected, (response.status_code, response.text[:300])
return response
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
"""Walk all pages; returns (items, number_of_pages)."""
items, pages, cursor = [], 0, None
while True:
query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "")
page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200,
token=actor["token"], schema="MessagePage").json()
pages += 1
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items, pages
assert len(page["items"]) == limit, "a non-final page must be full"
assert len(items) < 100_000, "cursor loop"
def _stand_is_up() -> bool:
try:
return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200
except httpx.HTTPError:
return False
def pytest_collection_modifyitems(config, items):
if any("stack" in item.keywords for item in items) and not _stand_is_up():
skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait")
for item in items:
if "stack" in item.keywords:
item.add_marker(skip)
@pytest.fixture(scope="session")
def api() -> Api:
return Api()
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
script = str(ROOT / "scripts" / name)
if os.name == "nt":
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
else:
command = [script]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
timeout=timeout, env={**os.environ, **env})
def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str:
"""Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3
credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is
not published on the host; this is how the tests inject faults directly into the bucket."""
script = ("import asyncio, base64, json, sys\n"
"from messenger.config import Settings, s3_settings\n"
"from messenger.storage import Storage, ObjectMissing\n"
"async def main():\n"
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
" s = Storage(st)\n"
" await s.open()\n"
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
" finally:\n"
" await s.close()\n"
"asyncio.run(main())\n")
if service == "api":
cmd = ["exec", "-T", "api", "python", "-c", script]
else:
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
def s3_put(key: str, data: bytes) -> None:
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})",
stdin=base64.b64encode(data).decode())
def s3_delete(key: str) -> None:
s3_exec(f"await s.delete({key!r})")
def s3_size(key: str) -> int | None:
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)")
size = int(out.splitlines()[-1])
return None if size < 0 else size
def s3_keys() -> set[str]:
"""Every object key under o/ (needs List: runs with the reconciler's key pair)."""
out = s3_exec("keys, after = [], None\n"
"while True:\n"
" page, more = await s.list_page('o/', after, 1000)\n"
" keys += [o.key for o in page]\n"
" if not more or not page:\n"
" break\n"
" after = page[-1].key\n"
"print(json.dumps(keys))", service="reconciler")
return set(json.loads(out.splitlines()[-1]))
def reconcile(*args: str, expect_code: int = 0) -> dict:
"""Run the reconciler CLI (compose service, profile tools) and return its JSON report."""
# --no-deps: never let `compose run` start a store that a test has stopped on purpose.
result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False,
timeout=300)
assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:])
return json.loads(result.stdout)
def findings_by_key(report: dict) -> dict[str, dict]:
return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items}
def attachment_row(attachment_id: str) -> dict | None:
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
"'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id)
return json.loads(out) if out else None
def probe_inside(replica: dict, path: str) -> int:
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
code = ("import urllib.request as u, urllib.error as e\n"
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
"except e.HTTPError as x: print(x.code)") % path
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
timeout=30).stdout.strip()
return int(out or 0)
+93
View File
@@ -0,0 +1,93 @@
"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the
reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without
scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file
is generated in memory."""
from __future__ import annotations
import io
import os
import struct
import zlib
from PIL import Image
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
def chunk(ctype: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data))
def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes:
return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace))
def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes:
if noise:
channels = len(mode)
image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels))
else:
image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128)
out = io.BytesIO()
image.save(out, "PNG", **save)
return out.getvalue()
def png_of_size(target: int) -> bytes:
"""A valid PNG of (about, at most) `target` bytes: random pixels do not compress."""
side = int((target / 3) ** 0.5)
data = png(side, side, noise=True, compress_level=1)
while len(data) > target:
side -= 8
data = png(side, side, noise=True, compress_level=1)
return data
def png_exactly(size: int) -> bytes:
"""A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk."""
base = png_of_size(size - 64)
pad = size - len(base) - 12
assert pad >= 0
return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:]
def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes:
if noise:
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
else:
image = Image.new("RGB", (width, height), (200, 30, 30))
out = io.BytesIO()
image.save(out, "JPEG", progressive=progressive, quality=quality)
return out.getvalue()
# ------------------------------------------------------------------ broken files
def png_corrupt_idat() -> bytes:
"""Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case)."""
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"")
def png_short_pixels() -> bytes:
"""A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises."""
raw = b"".join(b"\x00" + bytes(60) for _ in range(19))
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes:
"""Header of a huge picture: must be refused (413) from the first 33 bytes."""
raw_row = b"\x00" + bytes(width * 3)
body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far
return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"")
def jpeg_without_scan_data() -> bytes:
data = jpeg(128, 96)
sos = data.index(b"\xff\xda")
length = struct.unpack(">H", data[sos + 2:sos + 4])[0]
return data[:sos + 2 + length] + b"\xff\xd9"
def jpeg_bomb_header() -> bytes:
"""SOF0 announcing 20000 x 20000 = 400 Mpx."""
return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000)
+ b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9")
+344
View File
@@ -0,0 +1,344 @@
"""Contract behaviour over real HTTP (stand must be running). Lab 1 checks moved to Bearer + new lab 2 auth."""
from __future__ import annotations
import base64
import uuid
import pytest
from conftest import PASSWORD, basic
pytestmark = pytest.mark.stack
def key(item: dict) -> tuple[str, int]:
return item["created_at"], uuid.UUID(item["id"]).int
# ================================================================== registration & login
def test_register_never_returns_password_and_login_returns_bearer(api):
actor = api.register("alice", display_name="Алиса 👩‍💻")
assert set(actor["user"]) == {"id", "username", "display_name", "created_at"}
session = api.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], PASSWORD)}, schema="Session").json()
assert session["token_type"] == "Bearer" and len(session["token"]) >= 43
assert actor["username"] not in session["token"] and actor["id"] not in session["token"]
assert api.call("GET", "/api/v1/users/me", 200, token=session["token"], schema="User").json() == actor["user"]
def test_duplicate_username_is_409(api):
actor = api.register("dup")
body = {"username": actor["username"], "display_name": "Другой", "password": PASSWORD}
assert api.call("POST", "/api/v1/users", 409, json_body=body).json()["error"]["code"] == "conflict"
@pytest.mark.parametrize("password", ["short-11ch", " " * 12, " \t \n " * 3, "я" * 129])
def test_bad_password_on_registration_is_400(api, password):
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": password}
api.call("POST", "/api/v1/users", 400, json_body=body)
@pytest.mark.parametrize("password", ["я" * 128, "with:colons:inside", "Twelve chars"])
def test_edge_passwords_register_and_login(api, password):
actor = api.user("edge", password=password)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_wrong_password_and_unknown_user_give_the_same_401(api):
actor = api.register("victim")
wrong = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(actor["username"], "Wrong-password-1")})
unknown = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("nobody_" + uuid.uuid4().hex[:8], PASSWORD)})
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
assert wrong.headers["www-authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
@pytest.mark.parametrize("header", ["Basic é", "Basic !!!", "Basic " + base64.b64encode(b"no_colon").decode(),
"Basic " + base64.b64encode(b"\xff\xfe:x").decode(), "Bearer " + "a" * 43, ""])
def test_malformed_basic_is_401_not_500(api, header):
# Raw UTF-8 bytes, as curl would send them (httpx refuses non-ASCII str headers).
api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": header.encode("utf-8")} if header else {})
def test_logout_revokes_only_this_token(api):
actor = api.user("logout")
second = api.login(actor)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=actor["token"])
api.call("GET", "/api/v1/users/me", 200, token=second) # other session of the same user survives
def test_x_user_id_basic_and_garbage_do_not_authenticate(api):
actor = api.user("hdr")
for headers in ({"X-User-Id": actor["id"]},
{"Authorization": basic(actor["username"], PASSWORD)},
{"Authorization": "Bearer " + "A" * 43},
{"Authorization": "Bearer"}):
api.call("GET", "/api/v1/users/me", 401, headers=headers)
me = api.call("GET", "/api/v1/users/me", 200, token=actor["token"], headers={"X-User-Id": str(uuid.uuid4())})
assert me.json()["id"] == actor["id"]
# ================================================================== validation / errors (from lab 1)
@pytest.mark.parametrize("raw", [b"{", b"", b"[]", b"null", b'{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}',
b'{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}',
b'{"username": "abc_def", "display_name": "\\ud800", "password": "Twelve-chars"}'])
def test_malformed_json_is_400(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
@pytest.mark.parametrize("body", [
{"username": "abc_def", "display_name": "x"}, # no password
{"username": "abc_def", "display_name": "x", "password": "Twelve-chars", "extra": 1},
{"username": "Abc_def", "display_name": "x", "password": "Twelve-chars"},
{"username": "abc_def", "display_name": " ", "password": "Twelve-chars"},
{"username": "abc_def", "display_name": "x" * 101, "password": "Twelve-chars"},
{"username": 5, "display_name": "x", "password": "Twelve-chars"},
])
def test_invalid_registration_fields_are_400(api, body):
api.call("POST", "/api/v1/users", 400, json_body=body)
def test_unicode_limits_are_code_points_and_nul_is_stored(api):
actor = api.user("uni", display_name="😀" * 100)
assert actor["user"]["display_name"] == "😀" * 100
nul = api.user("nul", display_name="A\u0000B")
assert api.call("GET", "/api/v1/users/me", 200, token=nul["token"]).json()["display_name"] == "A\u0000B"
chat = api.chat(actor, title="T\u0000itle")
assert chat["title"] == "T\u0000itle"
assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world"
assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000
api.post_message(actor, chat, {"text": "я" * 4001}, 400)
texts = [m["text"] for m in api.history(actor, chat, 100)[0]]
assert texts == ["hello\u0000world", "🙂" * 4000]
def test_oversized_body_and_wrong_content_type(api):
api.call("POST", "/api/v1/users", 413, json_body={"username": "big_body", "display_name": "x" * 70_000, "password": PASSWORD})
api.call("POST", "/api/v1/users", 400, raw=b'{"username":"abc_def"}', headers={"Content-Type": "text/plain"})
# ================================================================== chats & membership
def test_chat_validation(api):
alice, bob = api.user("alice"), api.user("bob")
for body in ({"title": "x", "member_ids": [alice["id"], bob["id"], bob["id"]]},
{"title": "x", "member_ids": [alice["id"], str(uuid.uuid4())]},
{"title": "x", "member_ids": [bob["id"]]},
{"title": " ", "member_ids": [alice["id"]]},
{"title": "x", "member_ids": [alice["id"], "bob"]}):
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body=body)
def test_foreign_chat_is_indistinguishable_from_missing(api):
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
chat = api.chat(alice, bob)
for chat_id in (chat["id"], str(uuid.uuid4())):
base = f"/api/v1/chats/{chat_id}"
errors = [api.call("GET", base, 404, token=eve["token"]).json(),
api.call("GET", base + "/messages", 404, token=eve["token"]).json(),
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(),
api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()]
assert len({e["error"]["message"] for e in errors}) == 1
api.call("GET", "/api/v1/chats/not-a-uuid", 401)
api.call("GET", "/api/v1/chats/not-a-uuid", 400, token=alice["token"])
api.call("GET", "/api/v1/chats/NOT-A-UUID/messages", 400, token=alice["token"])
def test_chat_list_order_pagination_and_isolation(api):
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
created = [api.chat(alice, bob, title=f"Чат {i}") for i in range(5)]
api.chat(eve, title="Чат Евы")
walked, cursor = [], None
while True:
query = "?limit=2" + (f"&cursor={cursor}" if cursor else "")
page = api.call("GET", "/api/v1/chats" + query, 200, token=bob["token"], schema="ChatPage").json()
walked += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
assert walked == created
first = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?limit=1&cursor={first}", 400, token=bob["token"]) # cursor of another user
api.call("GET", "/api/v1/chats?limit=0", 400, token=bob["token"])
# ================================================================== messages & keyset pagination
def test_history_pages_without_loss_or_duplicates(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
sent = [api.send(alice if i % 2 else bob, chat, f"сообщение {i}") for i in range(25)]
items, pages = api.history(bob, chat, 7)
assert pages == 4 and items == sent and items == sorted(items, key=key)
assert sent[0]["sender_id"] == bob["id"] and sent[1]["sender_id"] == alice["id"]
@pytest.mark.parametrize("query", ["limit=0", "limit=101", "limit=abc", "limit=1&limit=2", "cursor=", "cursor=garbage",
"cursor=djF8bXx4fDF8Mg", "cursor=" + base64.urlsafe_b64encode(b"v1|m|x|99999999999999999999|y").decode()])
def test_bad_limit_or_cursor_is_400(api, query):
alice = api.user("alice")
chat = api.chat(alice)
api.call("GET", f"/api/v1/chats/{chat['id']}/messages?{query}", 400, token=alice["token"])
def test_cursor_bound_to_chat(api):
alice = api.user("alice")
a, b = api.chat(alice, title="A"), api.chat(alice, title="B")
for chat in (a, b):
api.send(alice, chat, "1")
api.send(alice, chat, "2")
cursor = api.call("GET", f"/api/v1/chats/{a['id']}/messages?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats/{b['id']}/messages?cursor={cursor}", 400, token=alice["token"])
empty = api.chat(alice, title="Empty")
api.call("GET", f"/api/v1/chats/{empty['id']}/messages?cursor=garbage", 400, token=alice["token"])
def test_read_after_write(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
for i in range(5):
message = api.send(alice, chat, f"m{i}")
last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1]
assert last == message
# ================================================================== lab 3: inputs found by the reviewer in other PRs
def b64(raw: bytes) -> str:
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
def bad_cursors(owner_id: str) -> list[str]:
"""Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time."""
good_uuid = str(uuid.uuid4())
return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"),
b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'),
b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()),
b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()),
b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()),
b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()),
b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())]
def test_malformed_cursor_values_are_400_on_both_lists(api):
alice = api.user("cur")
chat = api.chat(alice)
for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]):
quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20")
for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json()
assert error["error"]["code"] == "invalid_request", (path, cursor)
@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}',
b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}',
b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}',
b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}',
b'{"text":" \n\t "}'])
def test_bad_message_body_is_400_and_nothing_is_stored(api, raw):
alice = api.user("body")
chat = api.chat(alice)
error = api.post_message(alice, chat, None, 400, raw=raw).json()
assert error["error"]["code"] == "invalid_request"
assert api.history(alice, chat, 100)[0] == []
def test_ff_byte_body_on_registration_is_contract_error(api):
response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"})
assert response.json()["error"]["request_id"] == response.headers["x-request-id"]
def test_text_is_stored_verbatim_and_length_counts_raw_text(api):
alice = api.user("verbatim")
chat = api.chat(alice)
padded = " строка\nс пробелами "
assert api.send(alice, chat, padded)["text"] == padded
api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming
assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded]
def test_unknown_attachment_ids_are_404(api):
alice = api.user("att")
chat = api.chat(alice)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates
assert api.history(alice, chat, 100)[0] == []
def test_member_ids_are_compared_as_uuids_not_strings(api):
alice, bob = api.user("case"), api.user("case")
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]})
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())})
chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json()
assert chat["member_ids"] == [alice["id"], bob["id"]]
@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43])
def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header):
api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")})
def test_login_with_a_body_is_rejected(api):
actor = api.register("login_body")
auth = {"Authorization": basic(actor["username"], actor["password"])}
api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"})
api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth)
api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session")
def test_password_range_is_12_to_128_code_points(api):
api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"})
actor = api.user("pw128", password="я:" * 64)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_every_api_response_carries_instance_and_request_id(api):
"""X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it)."""
api.call("GET", "/health/live", 200)
api.call("GET", "/health/ready", 200)
api.call("GET", "/no/such/route", 404)
api.call("PUT", "/api/v1/users", 405)
api.call("GET", "/api/v1/users/me", 401)
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
@pytest.mark.parametrize("blank", [" ", "  ", "  \t "])
def test_unicode_whitespace_only_strings_are_400(api, blank):
alice = api.user("blank")
chat = api.chat(alice)
api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank,
"password": PASSWORD})
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]})
api.post_message(alice, chat, {"text": blank}, 400)
assert api.history(alice, chat, 100)[0] == []
@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}',
'{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}',
b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}'])
def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
api.call("GET", "/health/ready", 200)
def test_more_than_100_members_is_400(api):
owner = api.user("big")
others = [api.register(f"m{i}")["id"] for i in range(100)]
api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]})
chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json()
assert len(chat["member_ids"]) == 100
def test_cursor_of_one_list_is_rejected_by_the_other(api):
alice = api.user("cross")
chats = [api.chat(alice, title=f"c{i}") for i in range(2)]
for _ in range(2):
api.send(alice, chats[0], "m")
message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200,
token=alice["token"]).json()["next_cursor"]
chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"])
api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"])
+268
View File
@@ -0,0 +1,268 @@
"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits,
content checks, filename rules and messages with attachment_ids.
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
filename, text=null / missing text / attachment_ids=null in a message."""
from __future__ import annotations
import hashlib
import subprocess
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
MiB = 1024 * 1024
@pytest.fixture()
def trio(api):
"""alice + bob share a chat; eve is an outsider with her own chat."""
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
def content(api, actor, attachment_id, expected=200, query=""):
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
# ------------------------------------------------------------------ positive path
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
alice, bob, _, chat, _ = trio
jpg = imagegen.jpeg(321, 123, noise=True)
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json()
assert meta == att
got = content(api, bob, att["id"])
assert got.content == data and got.headers["content-type"] == ctype
assert got.headers["content-length"] == str(len(data))
assert got.headers["x-content-type-options"] == "nosniff"
assert got.headers["content-disposition"].startswith("attachment;")
assert "location" not in got.headers # bytes through the API, never a redirect to S3
def test_metadata_reveals_no_storage_details(api, trio):
alice, _, _, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["object_key"].startswith("o/")
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower()
# The key is random: neither the id nor the filename is part of it.
assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"]
assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket
def test_original_is_not_stored_on_api_containers(api, trio):
"""The API keeps no file of the original: no recently written file of that size anywhere in
the replica's filesystem (the bytes live only in the bucket)."""
alice, _, _, chat, _ = trio
data = imagegen.png_of_size(300_000)
api.upload(alice, chat, data)
size = len(data)
for cid in compose("ps", "-q", "api").stdout.split():
out = subprocess.run(
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
capture_output=True, text=True, timeout=60).stdout.strip()
assert out == "", out
# ------------------------------------------------------------------ access control
def test_outsider_and_unknown_get_404(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
for suffix in ("", "/content"):
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
# Upload into a foreign chat: 404 before anything about the body is looked at.
api.upload(eve, chat, SAMPLE, 404)
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
def test_attachment_ids_are_checked_per_chat(api, trio):
alice, bob, eve, chat, eve_chat = trio
other = api.chat(alice) # alice's second chat
att_other = api.upload(alice, other, SAMPLE).json()
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
# Alice is a member of both chats, but a file is bound to its chat.
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
# Eve cannot post into alice's chat even with her own file: membership first.
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
# Any member may reference an attachment of the chat.
att = api.upload(alice, chat, SAMPLE).json()
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
# ------------------------------------------------------------------ messages with attachments
def test_message_with_attachments_and_history(api, trio):
alice, bob, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
assert m2["text"] == " " and m3["attachment_ids"] == []
items, _ = api.history(bob, chat, 2)
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
def test_idempotent_replay_with_attachments(api, trio):
alice, _, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
# A failed attachment check does not burn the key: the same key works with a valid body later.
key2 = uuid.uuid4().hex
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
@pytest.mark.parametrize("raw", [
b'{"attachment_ids":["%(a)s"]}', # text missing
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
b'{"text":1,"attachment_ids":["%(a)s"]}',
b'{"text":"x","attachment_ids":null}', # attachment_ids null
b'{"text":"x","attachment_ids":"%(a)s"}',
b'{"text":"x","attachment_ids":{}}',
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
b'{"text":"x","attachment_ids":["%(a)s",1]}',
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
b'{"text":"","attachment_ids":[]}', # neither text nor files
b'{"text":" "}',
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
], ids=lambda raw: raw.decode()[:40])
def test_message_body_validation(api, trio, raw):
alice, _, _, chat, _ = trio
a = api.upload(alice, chat, SAMPLE).json()["id"]
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
def test_at_most_ten_attachments(api, trio):
alice, _, _, chat, _ = trio
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
# ------------------------------------------------------------------ limits
def test_size_limit_through_the_load_balancer(api, trio):
alice, bob, _, chat, _ = trio
exact = imagegen.png_exactly(10 * MiB)
assert len(exact) == 10 * MiB
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
assert content(api, bob, att["id"]).content == exact
started = time.monotonic()
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
assert time.monotonic() - started < 5
over = exact + bytes(1024)
def chunks(): # no Content-Length: chunked body
for i in range(0, len(over), 256 * 1024):
yield over[i:i + 256 * 1024]
error = api.upload(alice, chat, chunks(), 413).json()
assert error["error"]["code"] == "payload_too_large"
def test_pixel_limit_before_decoding(api, trio):
alice, _, _, chat, _ = trio
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
started = time.monotonic()
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
def test_exactly_40_megapixels_is_accepted(api, trio):
alice, _, _, chat, _ = trio
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
att = api.upload(alice, chat, data).json()
assert att["size_bytes"] == len(data)
# ------------------------------------------------------------------ content checks
BROKEN = {
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
"png: truncated": (SAMPLE[:-20], "image/png"),
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
"type: text/plain": (SAMPLE, "text/plain"),
"type: octet-stream": (SAMPLE, "application/octet-stream"),
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
"type: missing": (SAMPLE, None),
}
@pytest.mark.parametrize("name", list(BROKEN))
def test_bad_content_is_415(api, trio, name):
alice, _, _, chat, _ = trio
data, ctype = BROKEN[name]
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
assert error["error"]["code"] == "unsupported_media_type"
def test_empty_body_is_400(api, trio):
alice, _, _, chat, _ = trio
api.upload(alice, chat, b"", 400)
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("query", [
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
], ids=lambda q: q[:40] or "missing")
def test_bad_filename_is_400(api, trio, query):
alice, _, _, chat, _ = trio
api.upload(alice, chat, SAMPLE, 400, query=query)
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
alice, _, _, chat, _ = trio
name = 'отчёт "Q3"; ok.png'
att = api.upload(alice, chat, SAMPLE, filename=name).json()
assert att["filename"] == name
disposition = content(api, alice, att["id"]).headers["content-disposition"]
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
assert "filename*=UTF-8''" in disposition
# ------------------------------------------------------------------ variants
def test_variant_parameter(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4
content(api, alice, att["id"], 400, query="?variant=huge")
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed
+334
View File
@@ -0,0 +1,334 @@
"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
"""
from __future__ import annotations
import contextlib
import json
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
running_replicas, wait_healthy, wait_stand
pytestmark = pytest.mark.stack
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
# ------------------------------------------------------------------ level 3: replicas, no affinity
def test_one_session_is_served_by_every_real_replica(api):
actor = api.user("rr")
seen = instances_seen(api, actor["token"])
running = {r["hostname"] for r in running_replicas()}
assert len(running) == REPLICAS >= 2
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
# so there is no sticky session hiding split state.
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
assert switches >= len(seen) // 2, seen
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
def test_write_on_one_replica_is_read_on_the_other(api):
alice, bob = api.user("rw"), api.user("rw")
chat = api.chat(alice, bob)
for i in range(10):
sent = api.post_message(alice, chat, {"text": f"m{i}"})
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
assert read.json()["items"][-1] == sent.json()
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
assert r.headers["x-instance-id"]
assert logout.headers["x-instance-id"]
# ------------------------------------------------------------------ readiness drives the LB
def hba_file() -> str:
return psql("SHOW hba_file")
@contextlib.contextmanager
def isolated_from_postgres(replica: dict):
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
connections), while it keeps running and answering /health/live. Always undone."""
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
path = hba_file()
marker = "lab3-test-isolate"
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
try:
psql("SELECT pg_reload_conf()")
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
yield replica
finally:
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
psql("SELECT pg_reload_conf()")
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
wait_stand()
@contextlib.contextmanager
def disconnected_from_data_network(replica: dict):
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
LB but loses PostgreSQL AND Valkey. Always reconnected."""
network = next(n for n in json.loads(subprocess.run(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
try:
yield
finally:
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
wait_stand()
@pytest.mark.parametrize("how", ["pg_hba", "network"])
@pytest.mark.parametrize("position", range(REPLICAS))
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
503 -> HAProxy must take it out; no business request may reach it."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
actor = api.user("ready")
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
with isolate(replica):
started = time.monotonic()
while replica["hostname"] in lbctl.up_instances():
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
time.sleep(0.1)
detected = time.monotonic() - started
for _ in range(3): # the process stays responsive (event loop never blocked) ...
started_probe = time.monotonic()
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
seen = instances_seen(api, actor["token"], 40)
assert replica["hostname"] not in seen and len(seen) == 40
for i in range(20): # writes keep working, only through the ready replicas
assert api.register(f"r{i}")["id"]
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
# ------------------------------------------------------------------ stopping replicas under traffic
class Traffic:
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
def __init__(self, api: Api, workers: int = 4):
self.alice, self.bob = api.user("load"), api.user("load")
self.chat = api.chat(self.alice, self.bob)
self.workers = workers
self.stop = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
self.confirmed: list[dict] = []
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
def record(self, kind: str, status: int, instance: str | None) -> None:
with self.lock:
self.events.append((time.monotonic(), kind, status, instance))
def run(self, index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=3)
auth = {"Authorization": "Bearer " + self.alice["token"]}
n = 0
key, text = uuid.uuid4().hex, f"w{index}-0"
while not self.stop.is_set():
try:
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
headers={**auth, "Idempotency-Key": key})
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
if r.status_code == 201:
with self.lock:
self.confirmed.append(r.json())
n += 1
key, text = uuid.uuid4().hex, f"w{index}-{n}"
# any other outcome: keep key and text, retry the same request
except httpx.HTTPError:
self.record("POST", 0, None)
try:
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
except httpx.HTTPError:
self.record("GET", 0, None)
time.sleep(0.02)
client.close()
def __enter__(self) -> "Traffic":
for t in self.threads:
t.start()
time.sleep(1.0)
return self
def __exit__(self, *exc) -> None:
self.stop.set()
for t in self.threads:
t.join(timeout=10)
def failures(self, since: float = 0.0) -> list[tuple]:
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
def check_history(self, api: Api) -> None:
history = api.history(self.bob, self.chat, 100)[0]
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
@pytest.mark.parametrize("position", range(REPLICAS))
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
started = time.monotonic()
with replica_stopped(replica, "stop"):
stopped_after = time.monotonic() - started
time.sleep(3)
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
assert after and replica["hostname"] not in {e[3] for e in after}
time.sleep(1)
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
assert traffic.failures() == [], traffic.failures()[:10]
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
assert state["Status"] == "running" # restored by replica_stopped
traffic.check_history(api)
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
@pytest.mark.parametrize("position", range(REPLICAS))
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
killed_at = time.monotonic()
with replica_stopped(replica, "kill"):
time.sleep(4)
failures = traffic.failures(killed_at)
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
assert late and replica["hostname"] not in {e[3] for e in late}
assert last_failure <= 10
traffic.check_history(api)
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
def test_graceful_shutdown_order_in_logs(api):
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
with replica_stopped(replica, "stop"):
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
text=True, encoding="utf-8").stdout
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
names = [e["event"] for e in events]
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
t = {e["event"]: e["ts"] for e in events}
assert t["drain_started"] < t["shutdown_requested"]
assert state["ExitCode"] == 0 and not state["OOMKilled"]
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
# ------------------------------------------------------------------ discovery: restart and scaling
def test_full_restart_lb_rediscovers_all_replicas(api):
actor = api.user("restart")
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
run_hook("restart.sh")
wait_healthy("postgres", "valkey", "api")
wait_stand()
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
def test_scale_up_and_down_is_picked_up_without_errors(api):
with Traffic(api, workers=2) as traffic:
try:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
timeout=180)
lbctl.wait_up(REPLICAS + 1, timeout=30)
actor = api.user("scale")
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
timeout=180)
wait_stand()
time.sleep(1)
assert traffic.failures() == [], traffic.failures()[:10]
traffic.check_history(api)
# ------------------------------------------------------------------ level 4: deployment policy
def test_compose_policy_limits_ports_secrets_images():
config = json.loads(compose("config", "--format", "json").stdout)
services = config["services"]
# The reconciler (profile "tools") is started on demand and not part of the running stand.
assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"}
for name, svc in services.items():
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
if name in ("api", "migrate"):
assert "build" in svc # built from this repository's source
elif name == "s3-init":
assert svc["image"] == services["api"]["image"] # the same locally built image
else:
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"])
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key']
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys,
"migrate": ["database_url"], "s3-init": s3_keys, "lb": [],
# the API: its own S3 key pair only (no admin, no reconciler keys)
"api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]}
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [],
"s3-init": [], "api": [], "lb": []}
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"],
"s3-init": ["storage"], "api": ["data", "edge", "storage"],
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
assert config["networks"]["data"].get("internal") is True
assert config["networks"]["storage"].get("internal") is True
def test_running_containers_have_limits_and_no_bypass_ports():
ids = compose("ps", "-q").stdout.split()
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
name = item["Name"]
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
bindings = item["HostConfig"]["PortBindings"] or {}
if "-lb-" in name:
assert all(b["HostIp"] in ("127.0.0.1", "::1") for v in bindings.values() for b in v) # loopback only
else:
assert not bindings, f"{name} publishes ports in bypass of the LB"
def test_migrations_ran_once_by_the_migrate_service():
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"]
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
api_logs = compose("logs", "--no-log-prefix", "api").stdout
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate
+111
View File
@@ -0,0 +1,111 @@
"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4).
The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between
the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API
raises 503 at that exact point, as if the process or the network failed there):
* after_s3_put object written, row still pending, client got 503 -> the janitor removes
the object and the row after PENDING_GRACE_SECONDS (here 3 s);
* after_ready_commit row ready and object written, but the answer is lost (client got 503)
-> nothing may be deleted: the attachment stays intact and downloadable.
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
from __future__ import annotations
import socket
import time
from pathlib import Path
import pytest
import imagegen
from conftest import api_with_env, attachment_row, psql, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'")
return [tuple(line.split()) for line in out.splitlines() if line]
def wait_for(condition, timeout: float, what: str) -> None:
deadline = time.monotonic() + timeout
while not condition():
assert time.monotonic() < deadline, what
time.sleep(0.5)
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
sock.sendall(head.encode() + body)
if wait_answer is None:
sock.close()
return b""
time.sleep(wait_answer)
answer = sock.recv(65536)
sock.close()
return answer
def test_interrupted_and_stalled_uploads_leave_nothing(api):
alice = api.user("alice")
chat = api.chat(alice)
data = imagegen.png_of_size(2_000_000)
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
assert b"x-instance-id" in answer.lower()
time.sleep(1)
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api):
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
assert s3_size(key) is None # object removed first
# Normal configuration again: an upload in the same chat works and is not touched.
att = api.upload(alice, chat, SAMPLE).json()
assert attachment_row(att["id"])["status"] == "ready"
def test_lost_answer_after_commit_never_deletes_the_object(api):
"""The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API
'compensated' by deleting the object -> ready row without bytes. Here nothing compensates
in the request path, and the janitor only touches 'pending' rows."""
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "ready"
time.sleep(6) # several janitor rounds
assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE)
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
assert got.content == SAMPLE
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
time.sleep(4)
[(_, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE)
+117
View File
@@ -0,0 +1,117 @@
"""Database-level guarantees, checked over HTTP and verified directly in PostgreSQL / Valkey."""
from __future__ import annotations
import threading
import time
import uuid
import pytest
from conftest import PASSWORD, Api, compose, psql
pytestmark = pytest.mark.stack
def run_concurrently(workers: int, target) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
local = Api() # one HTTP client (connection) per worker
barrier.wait() # all workers fire at the same moment
try:
results[index] = target(local, index)
except BaseException as exc: # noqa: BLE001 - reported by the test
results[index] = exc
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_registration_race_exactly_one_201_and_seven_409():
username = "race_" + uuid.uuid4().hex[:10]
def register(client: Api, _index: int) -> int:
body = {"username": username, "display_name": "Гонка", "password": PASSWORD}
response = client.client.post("/api/v1/users", json=body)
return response.status_code
statuses = run_concurrently(8, register)
# Real HTTP codes are compared: a timeout, 500 or exception fails the test instead of counting as 409.
assert sorted(statuses, key=str) == [201] + [409] * 7, statuses
assert psql(f"SELECT count(*) FROM users WHERE username = '{username}'") == "1"
def _ins_counter() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
def test_create_chat_rolls_back_after_the_chat_row_was_inserted(api):
"""The chat INSERT runs first; the unknown member then breaks the membership INSERT (FK).
Proof that the first INSERT really happened and was rolled back: PostgreSQL counts
attempted inserts in pg_stat_user_tables.n_tup_ins even for aborted transactions,
while the table contains no such chat and no new membership rows.
"""
alice = api.user("rollback")
title = "rollback-" + uuid.uuid4().hex
members_before = int(psql("SELECT count(*) FROM chat_members"))
attempts_before = _ins_counter()
error = api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}).json()
assert "несуществующих" in error["error"]["message"]
deadline = time.monotonic() + 20 # statistics are flushed by the API backend asynchronously
while _ins_counter() == attempts_before and time.monotonic() < deadline:
time.sleep(0.5)
assert _ins_counter() > attempts_before, "the chat INSERT was never attempted"
assert psql(f"SELECT count(*) FROM chats WHERE title = convert_to('{title}', 'UTF8')") == "0"
assert int(psql("SELECT count(*) FROM chat_members")) == members_before
def test_migrations_rerun_is_a_noop():
users_before = psql("SELECT count(*) FROM users")
result = compose("run", "--rm", "--no-deps", "migrate")
assert "0001_init: already applied, skipped" in result.stdout
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
assert "0003_post_message_function: already applied, skipped" in result.stdout
assert "0004_attachments: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "4"
assert psql("SELECT count(*) FROM users") == users_before
def test_100_messages_from_10_clients_keep_order_and_cursor_walk(api):
clients = [api.user(f"w{i}") for i in range(10)]
chat = api.chat(clients[0], *clients[1:])
def writer(client: Api, index: int) -> list[dict]:
return [client.send(clients[index], chat, f"c{index}-m{j}") for j in range(10)]
results = run_concurrently(10, writer)
assert all(isinstance(r, list) for r in results), results
confirmed = [m for batch in results for m in batch]
history, pages = api.history(clients[5], chat, 7)
ids = [m["id"] for m in history]
assert len(confirmed) == 100 and len(history) == 100 and len(set(ids)) == 100 and pages == 15
assert set(ids) == {m["id"] for m in confirmed}
keys = [(m["created_at"], uuid.UUID(m["id"]).int) for m in history]
assert all(a < b for a, b in zip(keys, keys[1:])), "history is not strictly ordered"
stamps = [m["created_at"] for m in history]
assert len(set(stamps)) == 100, "per-chat clock must give unique timestamps"
for i, client in enumerate(clients):
own = [m["text"] for m in history if m["sender_id"] == client["id"]]
assert own == [f"c{i}-m{j}" for j in range(10)]
def test_secrets_are_not_stored_in_plain_text(api):
actor = api.user("store")
stored = psql(f"SELECT password_hash FROM users WHERE id = '{actor['id']}'")
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and PASSWORD not in stored
token_hex = actor["token"].encode().hex()
keys = compose("exec", "-T", "valkey", "sh", "-c",
'REDISCLI_AUTH="$(cat /run/secrets/valkey_password)" valkey-cli --scan --pattern "session:*"').stdout.split()
assert keys and all(len(k) == len("session:") + 64 for k in keys)
assert not any(actor["token"] in k or token_hex in k for k in keys)
+35
View File
@@ -0,0 +1,35 @@
"""The history index really bounds the work of a late page (>= 10 000 messages, generic plan)."""
from __future__ import annotations
import re
import sys
import pytest
from conftest import ROOT
pytestmark = pytest.mark.stack
sys.path.insert(0, str(ROOT / "scripts"))
import explain_history # noqa: E402
def section(text: str, number: int) -> str:
parts = re.split(r"^=== ", text, flags=re.M)
return next(p for p in parts if p.startswith(f"{number}."))
def buffers(plan: str) -> int:
return int(re.search(r"Buffers: shared hit=(\d+)", plan).group(1))
def test_late_page_uses_index_range_not_filter():
text = explain_history.run()
assert "20000" in text # the target chat holds 20 000 messages
app_late = section(text, 2)
assert "Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))" in app_late
assert "Rows Removed by Filter" not in app_late and "Sort" not in app_late
rejected = section(text, 3)
assert "Rows Removed by Filter: 19950" in rejected
# The app query touches a handful of pages; the rejected variants touch the whole prefix.
assert buffers(app_late) <= 10 < buffers(rejected) and buffers(app_late) <= 10 < buffers(section(text, 4))
+111
View File
@@ -0,0 +1,111 @@
"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards.
Every test stops/pauses a real container and restores it in `finally` (service_down).
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
for a valid token. Liveness/readiness of the processes are checked inside each container.
"""
from __future__ import annotations
import subprocess
import time
import pytest
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
pytestmark = pytest.mark.stack
DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead
CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
def timed(api, *args, **kwargs):
started = time.monotonic()
response = api.call(*args, lb_error_ok=True, **kwargs)
return response, time.monotonic() - started
def processes_alive_but_not_ready() -> None:
for replica in running_replicas():
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503
@pytest.fixture
def prepared(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
message = api.send(alice, chat, "до отказа")
return alice, bob, chat, message
def check_intact(api, alice, bob, chat, message):
"""After recovery the SAME tokens work and confirmed data is unchanged."""
assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"]
items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"]
assert items == [message]
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_valkey_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("valkey", how):
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_postgres_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("postgres", how):
for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
response, took = timed(api, "GET", path, 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable", path
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
def test_locked_table_gives_bounded_503(api, prepared):
"""A statement stuck behind a lock is stopped by lock_timeout, not after minutes."""
alice, bob, chat, message = prepared
locker = subprocess.Popen(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
time.sleep(1.5) # let the lock be taken
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
finally:
locker.wait(timeout=30)
check_intact(api, alice, bob, chat, message)
def test_logout_during_valkey_outage_is_never_lost(api, prepared):
"""Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten':
during the outage logout answers 503 (nothing revoked, the client knows it failed);
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
alice, bob, chat, message = prepared
with service_down("valkey", "stop"):
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
restart_services("valkey", "api")
for _ in range(10): # repeated checks: the revoked token must never come back
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"})
api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart
+281
View File
@@ -0,0 +1,281 @@
"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
undone in `finally`.
"""
from __future__ import annotations
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
pytestmark = pytest.mark.stack
def messages_in_db(chat: dict) -> int:
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
def key_rows(actor: dict, key: str) -> int:
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
@pytest.fixture
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
alice, bob, chat = pair
key = "replay-" + uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
served = {first.headers["x-instance-id"]}
for _ in range(20):
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
assert again.status_code == 201 and again.json() == first.json()
served.add(again.headers["x-instance-id"])
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
assert api.history(bob, chat, 100)[0] == [first.json()]
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
original = api.send(alice, chat, "оригинал", key=key)
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
assert api.send(alice, chat, "оригинал", key=key) == original
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
alice, _, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
assert messages_in_db(chat) == 1
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
def test_missing_or_invalid_key_is_400(api, pair, value):
alice, _, chat = pair
raw_value = value.encode("utf-8")
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
assert error["error"]["code"] == "invalid_request"
assert messages_in_db(chat) == 0
def test_repeated_key_header_is_400(pair):
alice, _, chat = pair
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
headers=[("Authorization", "Bearer " + alice["token"]),
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
assert response.status_code == 400 and response.headers["x-instance-id"]
assert messages_in_db(chat) == 0
def test_longest_key_and_every_allowed_character(api, pair):
alice, _, chat = pair
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
first = api.send(alice, chat, key[:20], key=key)
assert api.send(alice, chat, key[:20], key=key) == first
assert messages_in_db(chat) == 2
def test_scope_is_user_plus_chat_plus_key(api, pair):
alice, bob, chat = pair
other = api.chat(alice, bob, title="второй")
key = "shared-key"
a1 = api.send(alice, chat, "текст", key=key)
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
assert len({a1["id"], b1["id"], a2["id"]}) == 3
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
alice, bob, chat = pair
eve = api.user("eve")
key = uuid.uuid4().hex
api.send(alice, chat, "секрет", key=key)
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
assert messages_in_db(chat) == 1
def fire_together(workers: int, request) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
barrier.wait()
try:
results[index] = request(client, index)
except BaseException as exc: # noqa: BLE001 - asserted by the test
results[index] = exc
finally:
client.close()
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
alice, bob, chat = pair
key = "race-" + uuid.uuid4().hex
def post(client: httpx.Client, _index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.headers.get("x-instance-id"), r.json()
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
results = fire_together(24, post)
assert all(isinstance(r, tuple) for r in results), results
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_concurrent_same_key_different_bodies_one_wins(pair):
alice, _, chat = pair
key = "conflict-" + uuid.uuid4().hex
def post(client: httpx.Client, index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.json()
results = fire_together(20, post)
winners = [body for status, body in results if status == 201]
losers = [body for status, body in results if status == 409]
assert len(winners) + len(losers) == 20, results
assert winners and len({str(b) for b in winners}) == 1
assert all(b["error"]["code"] == "conflict" for b in losers)
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
assert messages_in_db(chat) == 1
def inserts_into_keys() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
transaction. Expected: 500 for the client, no key row and no message left behind
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
alice, bob, chat = pair
marker = "boom-" + uuid.uuid4().hex
key = uuid.uuid4().hex
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
"RETURN NEW; END $$; "
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
try:
before = inserts_into_keys()
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
while inserts_into_keys() == before and time.monotonic() < deadline:
time.sleep(0.5)
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
finally:
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
assert api.send(alice, chat, marker, key=key) == retried
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
f"SELECT pg_sleep({seconds}); COMMIT;")
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_lock_taken(chat: dict) -> None:
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
"AND pid <> pg_backend_pid()") != "0":
return
time.sleep(0.2)
raise AssertionError("the row lock was not taken")
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 1.0)
try:
wait_lock_taken(chat)
with pytest.raises(httpx.TimeoutException):
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
finally:
locker.wait(timeout=30)
deadline = time.monotonic() + 10
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
time.sleep(0.2)
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
assert stored, "the server-side transaction was expected to commit after the lock was released"
retried = api.send(alice, chat, "неопределённо", key=key)
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
nothing; after the lock is gone the same key creates the message exactly once."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 4.0)
try:
wait_lock_taken(chat)
started = time.monotonic()
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
assert time.monotonic() - started < 3.6
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
finally:
locker.wait(timeout=30)
created = api.send(alice, chat, "после блокировки", key=key)
assert api.send(alice, chat, "после блокировки", key=key) == created
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
def test_replay_after_the_serving_replica_is_stopped(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
served_by = first.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica):
for _ in range(5):
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
assert messages_in_db(chat) == 1
+217
View File
@@ -0,0 +1,217 @@
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
pairs from inside the storage network, PostgreSQL with psql):
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
other bytes (size / same-size checksum), object of a quarantined row restored;
DB side: stale intent with and without object, quarantine mark on a healthy row;
fresh: orphan and intent younger than --grace (an upload in progress).
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
the second run finds nothing to do; the safety valve and store outages stop the run without
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
from __future__ import annotations
import concurrent.futures
import hashlib
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
service_down)
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
GRACE = ["--grace", "8", "--allow-small-grace"]
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
def new_key() -> str:
return "o/" + uuid.uuid4().hex
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
attachment_id = str(uuid.uuid4())
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
"clock_timestamp() - make_interval(secs => %d))"
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
age_seconds))
return attachment_id
def snapshot() -> tuple[str, frozenset]:
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
return rows, frozenset(s3_keys())
def content_status(api, actor, attachment_id) -> int:
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
headers={"Authorization": "Bearer " + actor["token"]}).status_code
def test_detect_and_safely_fix_faults_on_both_sides(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for name, a in up.items()}
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
# ---- inject (old)
orphan_old = new_key()
s3_put(orphan_old, SAMPLE)
stale_with, stale_without = new_key(), new_key()
s3_put(stale_with, SAMPLE)
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"]) # ready + used, bytes lost
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
time.sleep(32) # everything above is now older than --grace
# ---- inject (fresh: an upload in progress)
fresh_orphan, fresh_intent = new_key(), new_key()
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
# ---- dry-run: report everything, change nothing
before = snapshot()
dry = reconcile(*LONG_GRACE)
assert snapshot() == before, "dry-run modified a store"
f = findings_by_key(dry)
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
key["D"]: "size_mismatch", key["E"]: "restored_object"}
assert {k: f[k]["kind"] for k in expected} == expected
assert key["A"] not in f and key["B"] not in f
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
# ---- apply
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
assert applied[k]["result"] == "applied", (k, applied[k])
keys = s3_keys()
assert orphan_old not in keys and stale_with not in keys
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
# fresh: untouched
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
# used, lost bytes: quarantined, NOT deleted; the message still references it
row_c = attachment_row(up["C"]["id"])
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
items, _ = api.history(bob, chat, 100)
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
assert content_status(api, bob, up["C"]["id"]) == 404
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
for name in "ABE":
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
assert got.content == data[name]
# ---- second run: idempotent, nothing left to do for these keys
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
assert k not in again, (k, again.get(k))
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
s3_put(key["C"], data["C"])
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
# tidy up the fresh items of this test once they are old enough
time.sleep(31)
reconcile("--apply", *LONG_GRACE)
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
def test_checksum_mode_finds_same_size_corruption(api):
alice = api.user("alice")
chat = api.chat(alice)
att = api.upload(alice, chat, SAMPLE).json()
k = attachment_row(att["id"])["object_key"]
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
time.sleep(9)
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
report = reconcile("--verify-checksums", *GRACE)
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays.
assert k not in findings_by_key(reconcile("--apply", *GRACE))
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# Restoring the original and verifying lifts it.
s3_put(k, SAMPLE)
lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]
assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied")
assert attachment_row(att["id"])["lost_reason"] is None
def test_safety_valve_refuses_mass_deletion():
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
--apply must stop before deleting anything."""
orphan = new_key()
s3_put(orphan, SAMPLE)
time.sleep(9)
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
assert orphan in s3_keys()
reconcile("--apply", *GRACE)
assert orphan not in s3_keys()
@pytest.mark.parametrize("service", ["s3", "postgres"])
def test_store_outage_stops_the_run_without_damage(api, service):
with service_down(service, "stop"):
report = reconcile(*GRACE, expect_code=1)
assert report["error"].startswith("store_unavailable")
assert reconcile(*GRACE)["error"] is None
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
ready and downloadable byte for byte, every message keeps its attachments."""
alice = api.user("alice")
chat = api.chat(alice)
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
done: list[tuple[dict, bytes]] = []
def worker(chunk: list[bytes]) -> None:
for p in chunk:
att = api.upload(alice, chat, p).json()
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
done.append((att, p))
with concurrent.futures.ThreadPoolExecutor(4) as pool:
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
runs = 0
while not all(x.done() for x in futures):
report = reconcile("--apply", *GRACE)
assert report["error"] is None
runs += 1
for x in futures:
x.result()
assert runs >= 1 and len(done) == len(payloads)
for att, p in done:
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
for att, p in done[::8]:
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p
+66
View File
@@ -0,0 +1,66 @@
"""Session lifetime: expiry with a short SESSION_TTL_SECONDS, no extension by restarts,
and the alternative SESSION_STORE=postgres. The api container is recreated with the
override and ALWAYS recreated back with the normal configuration (api_with_env)."""
from __future__ import annotations
import datetime as dt
import time
import pytest
from conftest import api_env, api_with_env, restart_services, service_down
pytestmark = pytest.mark.stack
def seconds_left(expires_at: str) -> float:
moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds()
def login_session(api, actor) -> dict:
from conftest import basic
return api.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json()
@pytest.mark.parametrize("store", ["valkey", "postgres"])
def test_short_ttl_expires(api, store):
with api_with_env(SESSION_TTL_SECONDS="2", SESSION_STORE=store):
assert api_env("SESSION_TTL_SECONDS") == "2" # the contract variable reaches the container
actor = api.register("ttl")
session = login_session(api, actor)
assert 1.0 < seconds_left(session["expires_at"]) <= 2.1
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=session["token"])
def test_restart_does_not_extend_ttl(api):
# 30 s: a restart of Valkey + both API replicas (readiness now also waits for S3) takes ~12-15 s.
with api_with_env(SESSION_TTL_SECONDS="30"):
actor = api.register("ttlrestart")
session = login_session(api, actor)
restart_services("valkey", "api")
assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it"
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
def test_postgres_session_store(api):
"""SESSION_STORE=postgres: sessions survive an API restart, logout revokes, and a DB outage
is 503 for a valid token (not 401 and not anonymous access)."""
with api_with_env(SESSION_STORE="postgres"):
assert api_env("SESSION_STORE") == "postgres"
actor = api.user("pgstore")
restart_services("api")
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
with service_down("postgres", "stop"):
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"],
lb_error_ok=True).json()["error"]["code"] == "unavailable"
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
+107
View File
@@ -0,0 +1,107 @@
"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4).
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
stand is waited for until HAProxy sees all replicas UP again."""
from __future__ import annotations
import time
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
@pytest.fixture()
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
"""S3 down (stopped: connection refused; paused: no answer at all):
* readiness is 503 on every replica (and through the LB), quickly;
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
* after recovery the earlier file is intact."""
alice, bob, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
with service_down("s3", how):
deadline = time.monotonic() + 15
while api.client.get("/health/ready").status_code != 503:
assert time.monotonic() < deadline, "readiness did not turn 503"
time.sleep(0.5)
for replica in running_replicas():
assert probe_inside(replica, "/health/ready") == 503
assert probe_inside(replica, "/health/ready/core") == 200
started = time.monotonic()
error = api.upload(alice, chat, SAMPLE, 503).json()
assert error["error"]["code"] == "unavailable"
assert time.monotonic() - started < 8
started = time.monotonic()
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
assert time.monotonic() - started < 8
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
api.send(alice, chat, "текст работает без S3") # degraded mode
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.content == SAMPLE
api.upload(alice, chat, SAMPLE)
def test_upload_survives_failover_of_the_serving_replica(api, pair):
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
(nothing of the file lives in the replica)."""
alice, bob, chat = pair
data = imagegen.png_of_size(500_000)
response = api.upload(alice, chat, data)
att, served_by = response.json(), response.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica, "kill"):
for _ in range(5):
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.headers["x-instance-id"] != served_by and got.content == data
def test_bucket_is_private_outside_the_api(api, pair):
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
unsigned LIST are refused; the same GET signed with the API key pair works (control).
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
alice, _, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
key = attachment_row(att["id"])["object_key"]
probe = (
"import urllib.request as u, urllib.error as e, socket\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"for port in (9333, 8888, 8080):\n"
" s = socket.socket(); s.settimeout(2)\n"
" print(port, s.connect_ex(('s3', port)))\n")
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
get_status, list_status = out[0].split()
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
for line in out[1:4]:
port, code = line.split()
assert code != "0", f"admin port {port} reachable"
signed = compose("exec", "-T", "api", "python", "-c",
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
assert signed == str(len(SAMPLE))
def test_s3_restart_keeps_objects(api, pair):
alice, bob, chat = pair
data = imagegen.png_of_size(200_000)
att = api.upload(alice, chat, data).json()
with service_down("s3", "stop"):
pass
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data
+148
View File
@@ -0,0 +1,148 @@
"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services.
The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it
(the same cases run against the stand in test_attachments.py)."""
from __future__ import annotations
import asyncio
import shutil
import sys
import time
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from messenger import images, validation # noqa: E402
from messenger.errors import InvalidRequest # noqa: E402
LIMIT = 40_000_000
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed")
def status(data: bytes) -> int:
"""201 if accepted, else the HTTP status of the ImageError (413/415)."""
try:
info = images.sniff(data, LIMIT)
if info is None:
return 415 # header never completed
asyncio.run(images.verify(data, info))
return 201
except images.ImageError as exc:
return exc.status
def test_public_fixture_is_accepted():
assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200)
assert status(SAMPLE) == 201
@pytest.mark.parametrize("data", [
imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"),
imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1),
], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"])
def test_valid_png_kinds(data):
assert status(data) == 201
def test_interlaced_png_is_accepted():
# Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0.
from messenger.images import _png_segments
raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1)
for _ in range(size // stride))
data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \
+ imagegen.chunk(b"IEND", b"")
assert status(data) == 201
BROKEN_PNG = {
"first 33 bytes of the fixture": SAMPLE[:33],
"truncated (no IEND)": SAMPLE[:-12],
"truncated in IDAT": SAMPLE[:len(SAMPLE) // 2],
"flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:],
"garbage IDAT with valid CRC": imagegen.png_corrupt_idat(),
"fewer rows than IHDR": imagegen.png_short_pixels(),
"bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip",
"text": b"just text, definitely not an image",
"gif": b"GIF89a" + bytes(64),
"two bytes of a signature": b"\x89P",
}
@pytest.mark.parametrize("name", list(BROKEN_PNG))
def test_broken_png_is_415(name):
assert status(BROKEN_PNG[name]) == 415, name
def test_pixel_limit_is_checked_from_the_header():
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough
assert exc.value.status == 413
exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed
assert images.sniff(exact, LIMIT).width == 8000
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT)
assert exc.value.status == 413
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.jpeg_bomb_header(), LIMIT)
assert exc.value.status == 413
def test_png_verification_is_streaming_and_fast():
"""40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded."""
big = imagegen.png(8000, 5000, compress_level=9)
started = time.perf_counter()
assert status(big) == 201
assert time.perf_counter() - started < 5
def test_jpeg_header_is_parsed_incrementally():
data = imagegen.jpeg(320, 200)
assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet
assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200)
@has_djpeg
@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"])
def test_valid_jpeg(data):
assert status(data) == 201
@has_djpeg
@pytest.mark.parametrize("name,data", [
("no scan data after SOS", imagegen.jpeg_without_scan_data()),
("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]),
], ids=lambda v: v if isinstance(v, str) else "")
def test_broken_jpeg_is_415(name, data):
assert status(data) == 415, name
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨‍👩‍👧.png", " x ", "..."[:2] + "x"])
def test_good_filenames(name):
assert validation.filename_param([("filename", name)]) == name
@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png",
"tab\t.png", "del\x7f.png", "x
y", "photo‮gnp.exe", "a" * 256])
def test_bad_filenames(name):
with pytest.raises(InvalidRequest):
validation.filename_param([("filename", name)])
def test_filename_must_be_single_and_present():
with pytest.raises(InvalidRequest):
validation.filename_param([])
with pytest.raises(InvalidRequest):
validation.filename_param([("filename", "a.png"), ("filename", "b.png")])
def test_query_must_be_valid_utf8():
assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")]
with pytest.raises(InvalidRequest):
validation.query_items(b"filename=%FF%FE.png")
+225
View File
@@ -0,0 +1,225 @@
"""Pure functions, no services: validation, cursor, Authorization parsing, Argon2id, config."""
from __future__ import annotations
import asyncio
import base64
import datetime as dt
import pytest
from messenger import authhdr, cursor, passwords, validation
from messenger.config import ConfigError, Settings
from messenger.errors import InvalidRequest, Unauthorized
from messenger.timefmt import UTC, from_us, rfc3339, to_us
def b64(raw: bytes) -> str:
return base64.b64encode(raw).decode()
# ------------------------------------------------------------------ password rules
@pytest.mark.parametrize("password", ["Twelve-chars", "я" * 128, "пароль:с:двоеточиями", "😀" * 12, "a b c d e f g"])
def test_valid_passwords(password):
assert validation.password_field({"password": password}) == password
@pytest.mark.parametrize("password", ["short-11ch", "я" * 129, " " * 12, "\t\n " * 5, "\ud800" * 12, None, 123456789012])
def test_invalid_passwords(password):
with pytest.raises(InvalidRequest):
validation.password_field({"password": password})
def test_text_may_contain_nul_but_not_lone_surrogates():
assert validation.text_field({"t": "a\u0000b"}, "t", 10) == "a\u0000b"
with pytest.raises(InvalidRequest):
validation.text_field({"t": "a\ud800"}, "t", 10)
# ------------------------------------------------------------------ cursor
CHAT = "3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11"
def test_cursor_roundtrip_and_binding():
key = (to_us(dt.datetime(2026, 10, 12, 12, 0, 0, 123456, tzinfo=UTC)), "11111111-1111-4111-8111-111111111111")
token = cursor.encode(cursor.MESSAGES, CHAT, key)
assert cursor.decode(token, cursor.MESSAGES, CHAT) == key
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.MESSAGES, "00000000-0000-4000-8000-000000000000") # other chat
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.CHATS, CHAT) # other resource
@pytest.mark.parametrize("payload", [
b"v1|m|" + CHAT.encode() + b"|253402300800000000|11111111-1111-4111-8111-111111111111", # 1 us after 9999-12-31
b"v1|m|" + CHAT.encode() + b"|9999999999999999999|11111111-1111-4111-8111-111111111111", # int64 edge
b"v1|m|" + CHAT.encode() + b"|-1|11111111-1111-4111-8111-111111111111",
b"v1|m|" + CHAT.encode() + b"|1|not-a-uuid",
b"v1|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-11111111111G",
b"v2|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-111111111111",
])
def test_broken_cursors_are_400_not_500(payload):
token = base64.urlsafe_b64encode(payload).rstrip(b"=").decode()
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.MESSAGES, CHAT)
def test_time_helpers_are_exact():
moment = dt.datetime(2026, 2, 28, 23, 59, 59, 999999, tzinfo=UTC)
assert from_us(to_us(moment)) == moment
assert rfc3339(moment) == "2026-02-28T23:59:59.999999Z"
assert from_us(cursor.MAX_TS_US).year == 9999
# ------------------------------------------------------------------ Authorization header
def test_basic_is_parsed_and_password_may_contain_colons():
assert authhdr.parse_basic("Basic " + b64("alice:pa:ss:wörd".encode())) == ("alice", "pa:ss:wörd")
assert authhdr.parse_basic("basic " + b64(b"bob_1:x" * 1)) == ("bob_1", "x")
@pytest.mark.parametrize("header", [
None, "", "Basic", "Basic é", "Basic !!!!", "Basic " + b64(b"no-colon"), "Basic " + b64(b"\xff\xfe:pw"),
"Basic " + b64(b":password"), "Basic " + b64(b"alice:"), "Basic " + b64("Алиса:pw".encode()),
"Bearer " + "a" * 43, "Basic " + "A" * 2000,
])
def test_bad_basic_is_401_with_basic_challenge(header):
with pytest.raises(Unauthorized) as info:
authhdr.parse_basic(header)
assert info.value.headers["WWW-Authenticate"].startswith("Basic realm=\"messenger\"")
@pytest.mark.parametrize("header", [None, "", "Bearer", "Bearer short", "Bearer " + "a" * 43 + "!",
"Basic " + b64(b"alice:password-123"), "Token " + "a" * 43, "Bearer " + "é" * 43])
def test_bad_bearer_is_401_with_bearer_challenge(header):
with pytest.raises(Unauthorized) as info:
authhdr.parse_bearer(header)
assert info.value.headers["WWW-Authenticate"] == 'Bearer realm="messenger"'
def test_bearer_ok():
token = "A_b-" + "x" * 39
assert authhdr.parse_bearer("Bearer " + token) == token
# ------------------------------------------------------------------ Argon2id
def test_argon2id_hash_and_verify():
async def scenario():
stored = await passwords.hash_password("я" * 128)
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$")
assert await passwords.verify_password(stored, "я" * 128)
assert not await passwords.verify_password(stored, "я" * 127)
assert not await passwords.verify_password(None, "anything") # unknown user, dummy hash
assert not await passwords.verify_password("not-a-hash", "anything")
assert await passwords.hash_password("same password!") != await passwords.hash_password("same password!") # salt
asyncio.run(scenario())
# ------------------------------------------------------------------ config
@pytest.fixture(autouse=True)
def s3_credentials(monkeypatch):
"""Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests."""
monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST")
monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests")
def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
settings = Settings.from_env()
assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000)
assert settings.fault_points == frozenset() and settings.upload_concurrency == 4
for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"),
("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")):
monkeypatch.setenv(name, value)
with pytest.raises(ConfigError):
Settings.from_env()
monkeypatch.delenv(name)
monkeypatch.delenv("S3_SECRET_ACCESS_KEY")
with pytest.raises(ConfigError):
Settings.from_env()
def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
url_file = tmp_path / "database_url"
url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")
monkeypatch.setenv("DATABASE_URL_FILE", str(url_file))
monkeypatch.setenv("SESSION_STORE", "redis")
monkeypatch.setenv("CACHE_URL", "redis://:p@valkey:6379/0")
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
settings = Settings.from_env()
assert settings.database_url == "postgresql://u:p@db/x"
assert settings.session_store == "valkey" and settings.session_ttl_seconds == 2
monkeypatch.setenv("SESSION_TTL_SECONDS", "0")
with pytest.raises(ConfigError):
Settings.from_env()
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "missing"))
with pytest.raises(ConfigError) as info:
Settings.from_env()
assert "u:p" not in str(info.value)
# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id
@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"])
def test_valid_idempotency_keys(key):
assert validation.idempotency_key(key) == key
@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"])
def test_invalid_idempotency_keys(key):
with pytest.raises(InvalidRequest):
validation.idempotency_key(key)
@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}'])
def test_duplicate_keys_and_garbage_json_are_rejected(raw):
with pytest.raises(InvalidRequest):
validation.parse_json_object(raw)
def test_attachment_ids_absent_equals_empty_and_is_validated():
assert validation.attachment_ids_field({"text": "x"}) == []
assert validation.attachment_ids_field({"attachment_ids": []}) == []
one = "11111111-1111-4111-8111-111111111111"
assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one]
for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11):
with pytest.raises(InvalidRequest):
validation.attachment_ids_field({"attachment_ids": bad})
def test_request_fingerprint_ignores_key_order_and_default_attachments():
from messenger.app import request_fingerprint
assert request_fingerprint("привет", []) == request_fingerprint("привет", [])
assert request_fingerprint("привет", []) != request_fingerprint("привет ", [])
assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", [])
one = "11111111-1111-4111-8111-111111111111"
assert request_fingerprint("x", [one]) != request_fingerprint("x", [])
def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch):
import socket
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.delenv("INSTANCE_ID", raising=False)
assert Settings.from_env().instance_id == socket.gethostname()
monkeypatch.setenv("INSTANCE_ID", "replica-a")
assert Settings.from_env().instance_id == "replica-a"
for bad in ("with space", "x" * 65, "ip/1"):
monkeypatch.setenv("INSTANCE_ID", bad)
with pytest.raises(ConfigError):
Settings.from_env()
def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600")
with pytest.raises(ConfigError):
Settings.from_env()
def test_message_write_path_setting(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.delenv("MESSAGE_WRITE_PATH", raising=False)
assert Settings.from_env().message_write_path == "function"
monkeypatch.setenv("MESSAGE_WRITE_PATH", "multi")
assert Settings.from_env().message_write_path == "multi"
monkeypatch.setenv("MESSAGE_WRITE_PATH", "fast")
with pytest.raises(ConfigError):
Settings.from_env()